{
  "schemaVersion": 1,
  "deviceProperties": [
    {
      "id": 0, "name": "NVIDIA H200", "totalGlobalMem": 149631664128,
      "computeMajor": 9, "computeMinor": 0,
      "maxThreadsPerBlock": 1024, "maxThreadsPerMultiprocessor": 2048,
      "regsPerBlock": 65536, "warpSize": 32,
      "sharedMemPerBlock": 49152, "numSms": 132
    , "regsPerMultiprocessor": 65536, "sharedMemPerBlockOptin": 232448, "sharedMemPerMultiprocessor": 233472
    },
    {
      "id": 1, "name": "NVIDIA H200", "totalGlobalMem": 149631664128,
      "computeMajor": 9, "computeMinor": 0,
      "maxThreadsPerBlock": 1024, "maxThreadsPerMultiprocessor": 2048,
      "regsPerBlock": 65536, "warpSize": 32,
      "sharedMemPerBlock": 49152, "numSms": 132
    , "regsPerMultiprocessor": 65536, "sharedMemPerBlockOptin": 232448, "sharedMemPerMultiprocessor": 233472
    },
    {
      "id": 2, "name": "NVIDIA H200", "totalGlobalMem": 149631664128,
      "computeMajor": 9, "computeMinor": 0,
      "maxThreadsPerBlock": 1024, "maxThreadsPerMultiprocessor": 2048,
      "regsPerBlock": 65536, "warpSize": 32,
      "sharedMemPerBlock": 49152, "numSms": 132
    , "regsPerMultiprocessor": 65536, "sharedMemPerBlockOptin": 232448, "sharedMemPerMultiprocessor": 233472
    },
    {
      "id": 3, "name": "NVIDIA H200", "totalGlobalMem": 149631664128,
      "computeMajor": 9, "computeMinor": 0,
      "maxThreadsPerBlock": 1024, "maxThreadsPerMultiprocessor": 2048,
      "regsPerBlock": 65536, "warpSize": 32,
      "sharedMemPerBlock": 49152, "numSms": 132
    , "regsPerMultiprocessor": 65536, "sharedMemPerBlockOptin": 232448, "sharedMemPerMultiprocessor": 233472
    },
    {
      "id": 4, "name": "NVIDIA H200", "totalGlobalMem": 149631664128,
      "computeMajor": 9, "computeMinor": 0,
      "maxThreadsPerBlock": 1024, "maxThreadsPerMultiprocessor": 2048,
      "regsPerBlock": 65536, "warpSize": 32,
      "sharedMemPerBlock": 49152, "numSms": 132
    , "regsPerMultiprocessor": 65536, "sharedMemPerBlockOptin": 232448, "sharedMemPerMultiprocessor": 233472
    },
    {
      "id": 5, "name": "NVIDIA H200", "totalGlobalMem": 149631664128,
      "computeMajor": 9, "computeMinor": 0,
      "maxThreadsPerBlock": 1024, "maxThreadsPerMultiprocessor": 2048,
      "regsPerBlock": 65536, "warpSize": 32,
      "sharedMemPerBlock": 49152, "numSms": 132
    , "regsPerMultiprocessor": 65536, "sharedMemPerBlockOptin": 232448, "sharedMemPerMultiprocessor": 233472
    },
    {
      "id": 6, "name": "NVIDIA H200", "totalGlobalMem": 149631664128,
      "computeMajor": 9, "computeMinor": 0,
      "maxThreadsPerBlock": 1024, "maxThreadsPerMultiprocessor": 2048,
      "regsPerBlock": 65536, "warpSize": 32,
      "sharedMemPerBlock": 49152, "numSms": 132
    , "regsPerMultiprocessor": 65536, "sharedMemPerBlockOptin": 232448, "sharedMemPerMultiprocessor": 233472
    },
    {
      "id": 7, "name": "NVIDIA H200", "totalGlobalMem": 149631664128,
      "computeMajor": 9, "computeMinor": 0,
      "maxThreadsPerBlock": 1024, "maxThreadsPerMultiprocessor": 2048,
      "regsPerBlock": 65536, "warpSize": 32,
      "sharedMemPerBlock": 49152, "numSms": 132
    , "regsPerMultiprocessor": 65536, "sharedMemPerBlockOptin": 232448, "sharedMemPerMultiprocessor": 233472
    }
  ],
  "cupti_version": 22,
  "cuda_runtime_version": 12040,
  "cuda_driver_version": 12080,
  "distributedInfo": {"backend": "nccl", "rank": 1, "world_size": 8, "pg_count": 1, "pg_config": [{"pg_name": "0", "pg_desc": "default_pg", "backend_config": "cuda:nccl", "pg_size": 8, "ranks": [0, 1, 2, 3, 4, 5, 6, 7]}], "nccl_version": "2.21.5"},
  "record_shapes": 1,
  "trace_id": "D09098EB50864776833F2FC2EE7B385F",
  "traceEvents": [
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: DivBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656707209.498, "dur": 146.786,
    "args": {
      "External id": 2483201,"Record function id": 0, "Sequence number": 24740650, "Fwd thread id": 1, "Ev Idx": 0
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "DivBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656707232.004, "dur": 113.079,
    "args": {
      "External id": 2483202,"Sequence number": 24740650, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[1]], "Ev Idx": 1
    }
  },
  {
    "ph": "f", "id": 1, "pid": 1336754, "tid": 1381179, "ts": 1514656707232.004,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::div", "pid": 1336754, "tid": 1381179,
    "ts": 1514656707242.480, "dur": 100.013,
    "args": {
      "External id": 2483203,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "long int"], "Input Strides": [[1], []], "Input Dims": [[1], []], "Ev Idx": 2
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: AddBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656707371.635, "dur": 300.957,
    "args": {
      "External id": 2483204,"Record function id": 0, "Ev Idx": 3
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656707450.479, "dur": 132.698,
    "args": {
      "External id": 2483205,"Record function id": 0, "Ev Idx": 4
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.27", "pid": 1336754, "tid": 1381179,
    "ts": 1514656707492.501, "dur": 77.468,
    "args": {
      "External id": 2483206,"Record function id": 0, "Ev Idx": 5
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "AddBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656707589.578, "dur": 2.483,
    "args": {
      "External id": 2483207,"Sequence number": 24740649, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[1]], "Ev Idx": 6
    }
  },
  {
    "ph": "f", "id": 2, "pid": 1336754, "tid": 1381179, "ts": 1514656707589.578,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1381179,
    "ts": 1514656707598.171, "dur": 67.860,
    "args": {
      "External id": 2483208,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", ""], "Input Strides": [[1], []], "Input Dims": [[1], []], "Ev Idx": 7
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1381179,
    "ts": 1514656707609.004, "dur": 56.546,
    "args": {
      "External id": 2483209,"Record function id": 0, "Concrete Inputs": ["", "[]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[1], [], [], []], "Ev Idx": 8
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656707621.662, "dur": 4.033,
    "args": {
      "External id": 2483210,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 9
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: FusedLinearCrossEntropyFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656707683.075, "dur": 36348.116,
    "args": {
      "External id": 2483211,"Record function id": 0, "Sequence number": 24740647, "Fwd thread id": 1, "Ev Idx": 10
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FusedLinearCrossEntropyFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656707685.143, "dur": 36331.967,
    "args": {
      "External id": 2483212,"Sequence number": 24740647, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 11
    }
  },
  {
    "ph": "f", "id": 3, "pid": 1336754, "tid": 1381179, "ts": 1514656707685.143,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656707744.591, "dur": 4.461,
    "args": {
      "External id": 2483213,"Record function id": 0, "Concrete Inputs": ["[]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 12
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514656707755.639, "dur": 36106.271,
    "args": {
      "External id": 2483214,"Record function id": 0, "Concrete Inputs": ["", "", "6", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 13
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514656707758.416, "dur": 36103.122,
    "args": {
      "External id": 2483215,"Record function id": 0, "Concrete Inputs": ["", "6", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], []], "Ev Idx": 14
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656707764.956, "dur": 9.230,
    "args": {
      "External id": 2483216,"Record function id": 0, "Concrete Inputs": ["[]", "[]", "6", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 15
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656707776.661, "dur": 36083.275,
    "args": {
      "External id": 2483217,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[], [], []], "Input Dims": [[], [], []], "Ev Idx": 16
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::lift_fresh", "pid": 1336754, "tid": 1381179,
    "ts": 1514656743867.030, "dur": 0.610,
    "args": {
      "External id": 2483218,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 17
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656743870.423, "dur": 3.760,
    "args": {
      "External id": 2483219,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 18
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656743872.380, "dur": 1.356,
    "args": {
      "External id": 2483220,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 19
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::ne", "pid": 1336754, "tid": 1381179,
    "ts": 1514656743881.261, "dur": 36.653,
    "args": {
      "External id": 2483221,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 20
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::is_nonzero", "pid": 1336754, "tid": 1381179,
    "ts": 1514656743926.569, "dur": 77.579,
    "args": {
      "External id": 2483222,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 21
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::item", "pid": 1336754, "tid": 1381179,
    "ts": 1514656743928.927, "dur": 51.956,
    "args": {
      "External id": 2483223,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 22
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_local_scalar_dense", "pid": 1336754, "tid": 1381179,
    "ts": 1514656743931.424, "dur": 49.147,
    "args": {
      "External id": 2483224,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 23
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744045.588, "dur": 24.520,
    "args": {
      "External id": 2483225,"Record function id": 0, "Sequence number": 24740646, "Fwd thread id": 1, "Ev Idx": 24
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744048.292, "dur": 18.629,
    "args": {
      "External id": 2483226,"Sequence number": 24740646, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 25
    }
  },
  {
    "ph": "f", "id": 4, "pid": 1336754, "tid": 1381179, "ts": 1514656744048.292,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744053.192, "dur": 13.451,
    "args": {
      "External id": 2483227,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 26
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744058.465, "dur": 7.963,
    "args": {
      "External id": 2483228,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 27
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SliceBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744074.246, "dur": 126.027,
    "args": {
      "External id": 2483229,"Record function id": 0, "Sequence number": 24740645, "Fwd thread id": 1, "Ev Idx": 28
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SliceBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744075.185, "dur": 114.348,
    "args": {
      "External id": 2483230,"Sequence number": 24740645, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 29
    }
  },
  {
    "ph": "f", "id": 5, "pid": 1336754, "tid": 1381179, "ts": 1514656744075.185,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice_backward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744078.413, "dur": 110.585,
    "args": {
      "External id": 2483231,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "2", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], [], []], "Ev Idx": 30
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744084.649, "dur": 42.038,
    "args": {
      "External id": 2483232,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 31
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744089.156, "dur": 6.651,
    "args": {
      "External id": 2483233,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 32
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744098.413, "dur": 27.904,
    "args": {
      "External id": 2483234,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 33
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744102.332, "dur": 23.526,
    "args": {
      "External id": 2483235,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 34
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744132.478, "dur": 5.237,
    "args": {
      "External id": 2483236,"Record function id": 0, "Concrete Inputs": ["", "2", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], []], "Ev Idx": 35
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744135.350, "dur": 1.886,
    "args": {
      "External id": 2483237,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[8388608, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], []], "Ev Idx": 36
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744138.807, "dur": 49.470,
    "args": {
      "External id": 2483238,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 37
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SelectBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744205.573, "dur": 70.538,
    "args": {
      "External id": 2483239,"Record function id": 0, "Sequence number": 24740644, "Fwd thread id": 1, "Ev Idx": 38
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SelectBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744206.654, "dur": 64.696,
    "args": {
      "External id": 2483240,"Sequence number": 24740644, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 39
    }
  },
  {
    "ph": "f", "id": 6, "pid": 1336754, "tid": 1381179, "ts": 1514656744206.654,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select_backward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744210.134, "dur": 60.965,
    "args": {
      "External id": 2483241,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "2", "3"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], []], "Ev Idx": 40
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744214.285, "dur": 23.682,
    "args": {
      "External id": 2483242,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 41
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744215.788, "dur": 3.608,
    "args": {
      "External id": 2483243,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 42
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744219.944, "dur": 17.742,
    "args": {
      "External id": 2483244,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 43
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744223.187, "dur": 14.130,
    "args": {
      "External id": 2483245,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], []], "Ev Idx": 44
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744242.425, "dur": 5.922,
    "args": {
      "External id": 2483246,"Record function id": 0, "Concrete Inputs": ["", "2", "3"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], []], "Input Dims": [[16, 4096, 4, 2048], [], []], "Ev Idx": 45
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744246.287, "dur": 1.278,
    "args": {
      "External id": 2483247,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "6144"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 46
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744249.485, "dur": 21.092,
    "args": {
      "External id": 2483248,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 47
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SliceBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744280.643, "dur": 142.390,
    "args": {
      "External id": 2483249,"Record function id": 0, "Sequence number": 24740643, "Fwd thread id": 1, "Ev Idx": 48
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SliceBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744282.071, "dur": 135.506,
    "args": {
      "External id": 2483250,"Sequence number": 24740643, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 49
    }
  },
  {
    "ph": "f", "id": 7, "pid": 1336754, "tid": 1381179, "ts": 1514656744282.071,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice_backward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744286.846, "dur": 130.360,
    "args": {
      "External id": 2483251,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "1", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], [], []], "Ev Idx": 50
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744289.687, "dur": 16.539,
    "args": {
      "External id": 2483252,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 51
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744290.554, "dur": 2.704,
    "args": {
      "External id": 2483253,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 52
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744293.767, "dur": 12.182,
    "args": {
      "External id": 2483254,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 53
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744294.905, "dur": 10.622,
    "args": {
      "External id": 2483255,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], []], "Ev Idx": 54
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744309.916, "dur": 2.549,
    "args": {
      "External id": 2483256,"Record function id": 0, "Concrete Inputs": ["", "1", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 55
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744311.452, "dur": 0.804,
    "args": {
      "External id": 2483257,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 56
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744313.489, "dur": 102.956,
    "args": {
      "External id": 2483258,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], [16, 4096, 4, 2048], []], "Ev Idx": 57
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SliceBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744429.795, "dur": 117.628,
    "args": {
      "External id": 2483259,"Record function id": 0, "Sequence number": 24740642, "Fwd thread id": 1, "Ev Idx": 58
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SliceBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744431.053, "dur": 111.548,
    "args": {
      "External id": 2483260,"Sequence number": 24740642, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 59
    }
  },
  {
    "ph": "f", "id": 8, "pid": 1336754, "tid": 1381179, "ts": 1514656744431.053,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice_backward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744433.429, "dur": 108.790,
    "args": {
      "External id": 2483261,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "0", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], [], []], "Ev Idx": 60
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744436.066, "dur": 20.046,
    "args": {
      "External id": 2483262,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 61
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744436.920, "dur": 4.771,
    "args": {
      "External id": 2483263,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 62
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744442.328, "dur": 13.538,
    "args": {
      "External id": 2483264,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 63
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744443.218, "dur": 12.311,
    "args": {
      "External id": 2483265,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], []], "Ev Idx": 64
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744457.406, "dur": 4.275,
    "args": {
      "External id": 2483266,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 65
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744460.893, "dur": 0.666,
    "args": {
      "External id": 2483267,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 66
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744465.228, "dur": 76.057,
    "args": {
      "External id": 2483268,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], [16, 4096, 4, 2048], []], "Ev Idx": 67
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: AddBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744553.871, "dur": 45.620,
    "args": {
      "External id": 2483269,"Record function id": 0, "Sequence number": 24740641, "Fwd thread id": 1, "Ev Idx": 68
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "AddBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744555.891, "dur": 1.352,
    "args": {
      "External id": 2483270,"Sequence number": 24740641, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[1]], "Ev Idx": 69
    }
  },
  {
    "ph": "f", "id": 9, "pid": 1336754, "tid": 1381179, "ts": 1514656744555.891,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744560.130, "dur": 34.517,
    "args": {
      "External id": 2483271,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", ""], "Input Strides": [[1], []], "Input Dims": [[1], []], "Ev Idx": 70
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744562.482, "dur": 31.568,
    "args": {
      "External id": 2483272,"Record function id": 0, "Concrete Inputs": ["", "[]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[1], [], [], []], "Ev Idx": 71
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744572.757, "dur": 0.682,
    "args": {
      "External id": 2483273,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 72
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: FusedLinearCrossEntropyFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744605.669, "dur": 2204.135,
    "args": {
      "External id": 2483274,"Record function id": 0, "Sequence number": 24740639, "Fwd thread id": 1, "Ev Idx": 73
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FusedLinearCrossEntropyFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744607.096, "dur": 2162.093,
    "args": {
      "External id": 2483275,"Sequence number": 24740639, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 74
    }
  },
  {
    "ph": "f", "id": 10, "pid": 1336754, "tid": 1381179, "ts": 1514656744607.096,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744652.727, "dur": 2.814,
    "args": {
      "External id": 2483276,"Record function id": 0, "Concrete Inputs": ["[]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 75
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744658.036, "dur": 2017.324,
    "args": {
      "External id": 2483277,"Record function id": 0, "Concrete Inputs": ["", "", "6", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 76
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744659.742, "dur": 2015.335,
    "args": {
      "External id": 2483278,"Record function id": 0, "Concrete Inputs": ["", "6", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], []], "Ev Idx": 77
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744662.314, "dur": 5.886,
    "args": {
      "External id": 2483279,"Record function id": 0, "Concrete Inputs": ["[]", "[]", "6", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 78
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656744669.263, "dur": 2004.981,
    "args": {
      "External id": 2483280,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[], [], []], "Input Dims": [[], [], []], "Ev Idx": 79
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::lift_fresh", "pid": 1336754, "tid": 1381179,
    "ts": 1514656746678.614, "dur": 0.532,
    "args": {
      "External id": 2483281,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 80
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656746683.274, "dur": 2.447,
    "args": {
      "External id": 2483282,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 81
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656746684.679, "dur": 0.899,
    "args": {
      "External id": 2483283,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 82
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::ne", "pid": 1336754, "tid": 1381179,
    "ts": 1514656746689.951, "dur": 23.534,
    "args": {
      "External id": 2483284,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 83
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::is_nonzero", "pid": 1336754, "tid": 1381179,
    "ts": 1514656746718.790, "dur": 42.681,
    "args": {
      "External id": 2483285,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 84
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::item", "pid": 1336754, "tid": 1381179,
    "ts": 1514656746719.770, "dur": 41.513,
    "args": {
      "External id": 2483286,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 85
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_local_scalar_dense", "pid": 1336754, "tid": 1381179,
    "ts": 1514656746721.038, "dur": 39.940,
    "args": {
      "External id": 2483287,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 86
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656746780.339, "dur": 24.114,
    "args": {
      "External id": 2483288,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 87
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656746817.566, "dur": 17.754,
    "args": {
      "External id": 2483289,"Record function id": 0, "Sequence number": 24740638, "Fwd thread id": 1, "Ev Idx": 88
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656746819.226, "dur": 12.608,
    "args": {
      "External id": 2483290,"Sequence number": 24740638, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 89
    }
  },
  {
    "ph": "f", "id": 11, "pid": 1336754, "tid": 1381179, "ts": 1514656746819.226,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656746826.318, "dur": 5.304,
    "args": {
      "External id": 2483291,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 90
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656746827.634, "dur": 3.844,
    "args": {
      "External id": 2483292,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 91
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SliceBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656746838.676, "dur": 75.024,
    "args": {
      "External id": 2483293,"Record function id": 0, "Sequence number": 24740637, "Fwd thread id": 1, "Ev Idx": 92
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SliceBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656746839.597, "dur": 67.630,
    "args": {
      "External id": 2483294,"Sequence number": 24740637, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 93
    }
  },
  {
    "ph": "f", "id": 12, "pid": 1336754, "tid": 1381179, "ts": 1514656746839.597,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice_backward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656746841.676, "dur": 65.073,
    "args": {
      "External id": 2483295,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "2", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], [], []], "Ev Idx": 94
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336754, "tid": 1381179,
    "ts": 1514656746846.362, "dur": 21.662,
    "args": {
      "External id": 2483296,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 95
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656746847.884, "dur": 3.426,
    "args": {
      "External id": 2483297,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 96
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656746852.066, "dur": 15.700,
    "args": {
      "External id": 2483298,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 97
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656746853.499, "dur": 13.902,
    "args": {
      "External id": 2483299,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 98
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1381179,
    "ts": 1514656746869.408, "dur": 6.256,
    "args": {
      "External id": 2483300,"Record function id": 0, "Concrete Inputs": ["", "2", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], []], "Ev Idx": 99
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656746871.664, "dur": 3.725,
    "args": {
      "External id": 2483301,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[8388608, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], []], "Ev Idx": 100
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656746876.573, "dur": 29.467,
    "args": {
      "External id": 2483302,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 101
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SelectBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656746919.827, "dur": 58.380,
    "args": {
      "External id": 2483303,"Record function id": 0, "Sequence number": 24740636, "Fwd thread id": 1, "Ev Idx": 102
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SelectBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656746920.979, "dur": 53.225,
    "args": {
      "External id": 2483304,"Sequence number": 24740636, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 103
    }
  },
  {
    "ph": "f", "id": 13, "pid": 1336754, "tid": 1381179, "ts": 1514656746920.979,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select_backward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656746925.433, "dur": 48.499,
    "args": {
      "External id": 2483305,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "2", "2"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], []], "Ev Idx": 104
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336754, "tid": 1381179,
    "ts": 1514656746927.215, "dur": 21.586,
    "args": {
      "External id": 2483306,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 105
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656746928.325, "dur": 2.474,
    "args": {
      "External id": 2483307,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 106
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656746931.665, "dur": 16.884,
    "args": {
      "External id": 2483308,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 107
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656746932.781, "dur": 15.411,
    "args": {
      "External id": 2483309,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], []], "Ev Idx": 108
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336754, "tid": 1381179,
    "ts": 1514656746949.836, "dur": 7.570,
    "args": {
      "External id": 2483310,"Record function id": 0, "Concrete Inputs": ["", "2", "2"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], []], "Input Dims": [[16, 4096, 4, 2048], [], []], "Ev Idx": 109
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656746955.884, "dur": 1.016,
    "args": {
      "External id": 2483311,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "4096"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 110
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656746958.229, "dur": 15.153,
    "args": {
      "External id": 2483312,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 111
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SliceBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656747008.878, "dur": 102.120,
    "args": {
      "External id": 2483313,"Record function id": 0, "Sequence number": 24740635, "Fwd thread id": 1, "Ev Idx": 112
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SliceBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656747012.959, "dur": 93.392,
    "args": {
      "External id": 2483314,"Sequence number": 24740635, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 113
    }
  },
  {
    "ph": "f", "id": 14, "pid": 1336754, "tid": 1381179, "ts": 1514656747012.959,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice_backward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656747015.474, "dur": 90.484,
    "args": {
      "External id": 2483315,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "1", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], [], []], "Ev Idx": 114
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336754, "tid": 1381179,
    "ts": 1514656747017.031, "dur": 21.258,
    "args": {
      "External id": 2483316,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 115
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656747018.301, "dur": 3.097,
    "args": {
      "External id": 2483317,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 116
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656747022.063, "dur": 15.971,
    "args": {
      "External id": 2483318,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 117
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656747023.075, "dur": 14.564,
    "args": {
      "External id": 2483319,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], []], "Ev Idx": 118
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1381179,
    "ts": 1514656747039.256, "dur": 4.733,
    "args": {
      "External id": 2483320,"Record function id": 0, "Concrete Inputs": ["", "1", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 119
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656747042.473, "dur": 1.260,
    "args": {
      "External id": 2483321,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 120
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656747044.746, "dur": 60.568,
    "args": {
      "External id": 2483322,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], [16, 4096, 4, 2048], []], "Ev Idx": 121
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SliceBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656747116.123, "dur": 97.783,
    "args": {
      "External id": 2483323,"Record function id": 0, "Sequence number": 24740634, "Fwd thread id": 1, "Ev Idx": 122
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SliceBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656747119.546, "dur": 74.766,
    "args": {
      "External id": 2483324,"Sequence number": 24740634, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 123
    }
  },
  {
    "ph": "f", "id": 15, "pid": 1336754, "tid": 1381179, "ts": 1514656747119.546,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice_backward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656747123.574, "dur": 70.409,
    "args": {
      "External id": 2483325,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "0", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], [], []], "Ev Idx": 124
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336754, "tid": 1381179,
    "ts": 1514656747124.582, "dur": 18.125,
    "args": {
      "External id": 2483326,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 125
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656747125.216, "dur": 2.160,
    "args": {
      "External id": 2483327,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 126
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656747128.016, "dur": 14.437,
    "args": {
      "External id": 2483328,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 127
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656747128.919, "dur": 13.145,
    "args": {
      "External id": 2483329,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], []], "Ev Idx": 128
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1381179,
    "ts": 1514656747143.547, "dur": 2.080,
    "args": {
      "External id": 2483330,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 129
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656747144.663, "dur": 0.653,
    "args": {
      "External id": 2483331,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 130
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656747146.216, "dur": 47.117,
    "args": {
      "External id": 2483332,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], [16, 4096, 4, 2048], []], "Ev Idx": 131
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656747198.075, "dur": 14.500,
    "args": {
      "External id": 2483333,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], [16, 4096, 4, 2048], []], "Ev Idx": 132
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: AddBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656747218.038, "dur": 41.608,
    "args": {
      "External id": 2483334,"Record function id": 0, "Sequence number": 24740633, "Fwd thread id": 1, "Ev Idx": 133
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "AddBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656747221.726, "dur": 1.261,
    "args": {
      "External id": 2483335,"Sequence number": 24740633, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[1]], "Ev Idx": 134
    }
  },
  {
    "ph": "f", "id": 16, "pid": 1336754, "tid": 1381179, "ts": 1514656747221.726,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1381179,
    "ts": 1514656747225.566, "dur": 29.304,
    "args": {
      "External id": 2483336,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", ""], "Input Strides": [[1], []], "Input Dims": [[1], []], "Ev Idx": 135
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1381179,
    "ts": 1514656747227.327, "dur": 27.046,
    "args": {
      "External id": 2483337,"Record function id": 0, "Concrete Inputs": ["", "[]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[1], [], [], []], "Ev Idx": 136
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656747234.413, "dur": 2.815,
    "args": {
      "External id": 2483338,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 137
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: FusedLinearCrossEntropyFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656747264.679, "dur": 3053.399,
    "args": {
      "External id": 2483339,"Record function id": 0, "Sequence number": 24740631, "Fwd thread id": 1, "Ev Idx": 138
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FusedLinearCrossEntropyFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656747267.835, "dur": 3020.727,
    "args": {
      "External id": 2483340,"Sequence number": 24740631, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 139
    }
  },
  {
    "ph": "f", "id": 17, "pid": 1336754, "tid": 1381179, "ts": 1514656747267.835,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656747299.122, "dur": 2.084,
    "args": {
      "External id": 2483341,"Record function id": 0, "Concrete Inputs": ["[]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 140
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514656747303.494, "dur": 2895.641,
    "args": {
      "External id": 2483342,"Record function id": 0, "Concrete Inputs": ["", "", "6", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 141
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514656747304.529, "dur": 2894.253,
    "args": {
      "External id": 2483343,"Record function id": 0, "Concrete Inputs": ["", "6", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], []], "Ev Idx": 142
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656747306.474, "dur": 5.467,
    "args": {
      "External id": 2483344,"Record function id": 0, "Concrete Inputs": ["[]", "[]", "6", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 143
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656747312.736, "dur": 2884.991,
    "args": {
      "External id": 2483345,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[], [], []], "Input Dims": [[], [], []], "Ev Idx": 144
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::lift_fresh", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750202.340, "dur": 0.349,
    "args": {
      "External id": 2483346,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 145
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750203.567, "dur": 2.158,
    "args": {
      "External id": 2483347,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 146
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750204.875, "dur": 0.737,
    "args": {
      "External id": 2483348,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 147
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::ne", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750209.489, "dur": 22.388,
    "args": {
      "External id": 2483349,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 148
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::is_nonzero", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750241.303, "dur": 40.199,
    "args": {
      "External id": 2483350,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 149
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::item", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750242.314, "dur": 39.023,
    "args": {
      "External id": 2483351,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 150
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_local_scalar_dense", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750243.427, "dur": 37.597,
    "args": {
      "External id": 2483352,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 151
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750297.607, "dur": 15.769,
    "args": {
      "External id": 2483353,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 152
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750328.641, "dur": 11.351,
    "args": {
      "External id": 2483354,"Record function id": 0, "Sequence number": 24740630, "Fwd thread id": 1, "Ev Idx": 153
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750330.300, "dur": 6.900,
    "args": {
      "External id": 2483355,"Sequence number": 24740630, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 154
    }
  },
  {
    "ph": "f", "id": 18, "pid": 1336754, "tid": 1381179, "ts": 1514656750330.300,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750332.992, "dur": 4.013,
    "args": {
      "External id": 2483356,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 155
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750333.979, "dur": 2.890,
    "args": {
      "External id": 2483357,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 156
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SliceBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750343.346, "dur": 68.542,
    "args": {
      "External id": 2483358,"Record function id": 0, "Sequence number": 24740629, "Fwd thread id": 1, "Ev Idx": 157
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SliceBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750344.272, "dur": 62.063,
    "args": {
      "External id": 2483359,"Sequence number": 24740629, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 158
    }
  },
  {
    "ph": "f", "id": 19, "pid": 1336754, "tid": 1381179, "ts": 1514656750344.272,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice_backward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750348.525, "dur": 57.348,
    "args": {
      "External id": 2483360,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "2", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], [], []], "Ev Idx": 159
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750353.062, "dur": 19.910,
    "args": {
      "External id": 2483361,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 160
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750354.458, "dur": 2.741,
    "args": {
      "External id": 2483362,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 161
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750357.869, "dur": 14.842,
    "args": {
      "External id": 2483363,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 162
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750359.528, "dur": 12.750,
    "args": {
      "External id": 2483364,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 163
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750374.279, "dur": 4.991,
    "args": {
      "External id": 2483365,"Record function id": 0, "Concrete Inputs": ["", "2", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], []], "Ev Idx": 164
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750378.270, "dur": 0.754,
    "args": {
      "External id": 2483366,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[8388608, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], []], "Ev Idx": 165
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750380.110, "dur": 25.035,
    "args": {
      "External id": 2483367,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 166
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SelectBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750415.959, "dur": 54.001,
    "args": {
      "External id": 2483368,"Record function id": 0, "Sequence number": 24740628, "Fwd thread id": 1, "Ev Idx": 167
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SelectBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750416.933, "dur": 49.813,
    "args": {
      "External id": 2483369,"Sequence number": 24740628, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 168
    }
  },
  {
    "ph": "f", "id": 20, "pid": 1336754, "tid": 1381179, "ts": 1514656750416.933,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select_backward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750418.610, "dur": 47.893,
    "args": {
      "External id": 2483370,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "2", "1"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], []], "Ev Idx": 169
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750420.027, "dur": 20.229,
    "args": {
      "External id": 2483371,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 170
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750423.155, "dur": 2.444,
    "args": {
      "External id": 2483372,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 171
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750426.157, "dur": 13.854,
    "args": {
      "External id": 2483373,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 172
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750426.716, "dur": 12.940,
    "args": {
      "External id": 2483374,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], []], "Ev Idx": 173
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750441.185, "dur": 9.276,
    "args": {
      "External id": 2483375,"Record function id": 0, "Concrete Inputs": ["", "2", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], []], "Input Dims": [[16, 4096, 4, 2048], [], []], "Ev Idx": 174
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750446.460, "dur": 3.561,
    "args": {
      "External id": 2483376,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "2048"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 175
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750451.015, "dur": 14.948,
    "args": {
      "External id": 2483377,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 176
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SliceBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750473.770, "dur": 112.281,
    "args": {
      "External id": 2483378,"Record function id": 0, "Sequence number": 24740627, "Fwd thread id": 1, "Ev Idx": 177
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SliceBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750474.936, "dur": 106.132,
    "args": {
      "External id": 2483379,"Sequence number": 24740627, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 178
    }
  },
  {
    "ph": "f", "id": 21, "pid": 1336754, "tid": 1381179, "ts": 1514656750474.936,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice_backward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750476.788, "dur": 103.871,
    "args": {
      "External id": 2483380,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "1", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], [], []], "Ev Idx": 179
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750477.747, "dur": 22.162,
    "args": {
      "External id": 2483381,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 180
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750478.447, "dur": 1.989,
    "args": {
      "External id": 2483382,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 181
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750483.041, "dur": 16.597,
    "args": {
      "External id": 2483383,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 182
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750483.698, "dur": 15.574,
    "args": {
      "External id": 2483384,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], []], "Ev Idx": 183
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750500.717, "dur": 4.503,
    "args": {
      "External id": 2483385,"Record function id": 0, "Concrete Inputs": ["", "1", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 184
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750504.377, "dur": 0.660,
    "args": {
      "External id": 2483386,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 185
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750505.834, "dur": 73.710,
    "args": {
      "External id": 2483387,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], [16, 4096, 4, 2048], []], "Ev Idx": 186
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SliceBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750592.036, "dur": 99.032,
    "args": {
      "External id": 2483388,"Record function id": 0, "Sequence number": 24740626, "Fwd thread id": 1, "Ev Idx": 187
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SliceBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750593.507, "dur": 79.707,
    "args": {
      "External id": 2483389,"Sequence number": 24740626, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 188
    }
  },
  {
    "ph": "f", "id": 22, "pid": 1336754, "tid": 1381179, "ts": 1514656750593.507,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice_backward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750597.186, "dur": 75.574,
    "args": {
      "External id": 2483390,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "0", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], [], []], "Ev Idx": 189
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750598.239, "dur": 19.832,
    "args": {
      "External id": 2483391,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 190
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750599.186, "dur": 2.219,
    "args": {
      "External id": 2483392,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 191
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750602.120, "dur": 15.704,
    "args": {
      "External id": 2483393,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 192
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750605.355, "dur": 12.133,
    "args": {
      "External id": 2483394,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], []], "Ev Idx": 193
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750618.946, "dur": 4.368,
    "args": {
      "External id": 2483395,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 194
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750622.536, "dur": 0.542,
    "args": {
      "External id": 2483396,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 195
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750623.767, "dur": 48.316,
    "args": {
      "External id": 2483397,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], [16, 4096, 4, 2048], []], "Ev Idx": 196
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750676.445, "dur": 13.502,
    "args": {
      "External id": 2483398,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], [16, 4096, 4, 2048], []], "Ev Idx": 197
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: AddBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750695.185, "dur": 31.899,
    "args": {
      "External id": 2483399,"Record function id": 0, "Sequence number": 24740625, "Fwd thread id": 1, "Ev Idx": 198
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "AddBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750696.261, "dur": 1.228,
    "args": {
      "External id": 2483400,"Sequence number": 24740625, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[1]], "Ev Idx": 199
    }
  },
  {
    "ph": "f", "id": 23, "pid": 1336754, "tid": 1381179, "ts": 1514656750696.261,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750698.987, "dur": 22.496,
    "args": {
      "External id": 2483401,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", ""], "Input Strides": [[1], []], "Input Dims": [[1], []], "Ev Idx": 200
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750700.648, "dur": 20.385,
    "args": {
      "External id": 2483402,"Record function id": 0, "Concrete Inputs": ["", "[]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[1], [], [], []], "Ev Idx": 201
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750704.980, "dur": 0.414,
    "args": {
      "External id": 2483403,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 202
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: FusedLinearCrossEntropyFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750731.688, "dur": 3100.195,
    "args": {
      "External id": 2483404,"Record function id": 0, "Sequence number": 24740624, "Fwd thread id": 1, "Ev Idx": 203
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FusedLinearCrossEntropyFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750744.407, "dur": 3053.590,
    "args": {
      "External id": 2483405,"Sequence number": 24740624, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 204
    }
  },
  {
    "ph": "f", "id": 24, "pid": 1336754, "tid": 1381179, "ts": 1514656750744.407,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750772.866, "dur": 1.923,
    "args": {
      "External id": 2483406,"Record function id": 0, "Concrete Inputs": ["[]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 205
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750776.877, "dur": 2934.736,
    "args": {
      "External id": 2483407,"Record function id": 0, "Concrete Inputs": ["", "", "6", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 206
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750778.211, "dur": 2933.167,
    "args": {
      "External id": 2483408,"Record function id": 0, "Concrete Inputs": ["", "6", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], []], "Ev Idx": 207
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750782.703, "dur": 3.067,
    "args": {
      "External id": 2483409,"Record function id": 0, "Concrete Inputs": ["[]", "[]", "6", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 208
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656750786.458, "dur": 2923.578,
    "args": {
      "External id": 2483410,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[], [], []], "Input Dims": [[], [], []], "Ev Idx": 209
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::lift_fresh", "pid": 1336754, "tid": 1381179,
    "ts": 1514656753714.587, "dur": 0.503,
    "args": {
      "External id": 2483411,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 210
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656753716.043, "dur": 2.757,
    "args": {
      "External id": 2483412,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 211
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656753717.688, "dur": 0.720,
    "args": {
      "External id": 2483413,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 212
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::ne", "pid": 1336754, "tid": 1381179,
    "ts": 1514656753722.413, "dur": 23.027,
    "args": {
      "External id": 2483414,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 213
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::is_nonzero", "pid": 1336754, "tid": 1381179,
    "ts": 1514656753750.035, "dur": 40.852,
    "args": {
      "External id": 2483415,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 214
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::item", "pid": 1336754, "tid": 1381179,
    "ts": 1514656753750.873, "dur": 39.838,
    "args": {
      "External id": 2483416,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 215
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_local_scalar_dense", "pid": 1336754, "tid": 1381179,
    "ts": 1514656753752.129, "dur": 38.335,
    "args": {
      "External id": 2483417,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 216
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656753809.257, "dur": 16.492,
    "args": {
      "External id": 2483418,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 217
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656753842.837, "dur": 14.316,
    "args": {
      "External id": 2483419,"Record function id": 0, "Ev Idx": 218
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656753845.553, "dur": 9.466,
    "args": {
      "External id": 2483420,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 219
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656753849.122, "dur": 4.672,
    "args": {
      "External id": 2483421,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 220
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656753850.053, "dur": 3.642,
    "args": {
      "External id": 2483422,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 221
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656753860.686, "dur": 12.897,
    "args": {
      "External id": 2483423,"Record function id": 0, "Sequence number": 24740623, "Fwd thread id": 1, "Ev Idx": 222
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656753861.942, "dur": 8.353,
    "args": {
      "External id": 2483424,"Sequence number": 24740623, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 223
    }
  },
  {
    "ph": "f", "id": 25, "pid": 1336754, "tid": 1381179, "ts": 1514656753861.942,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656753864.377, "dur": 5.722,
    "args": {
      "External id": 2483425,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 224
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656753867.568, "dur": 2.444,
    "args": {
      "External id": 2483426,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 225
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SliceBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656753876.982, "dur": 75.268,
    "args": {
      "External id": 2483427,"Record function id": 0, "Sequence number": 24740622, "Fwd thread id": 1, "Ev Idx": 226
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SliceBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656753877.985, "dur": 68.811,
    "args": {
      "External id": 2483428,"Sequence number": 24740622, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 227
    }
  },
  {
    "ph": "f", "id": 26, "pid": 1336754, "tid": 1381179, "ts": 1514656753877.985,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice_backward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656753879.734, "dur": 66.610,
    "args": {
      "External id": 2483429,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "2", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], [], []], "Ev Idx": 228
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336754, "tid": 1381179,
    "ts": 1514656753884.465, "dur": 22.447,
    "args": {
      "External id": 2483430,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 229
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656753885.510, "dur": 3.148,
    "args": {
      "External id": 2483431,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 230
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656753889.618, "dur": 17.000,
    "args": {
      "External id": 2483432,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 231
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656753890.965, "dur": 15.172,
    "args": {
      "External id": 2483433,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 232
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1381179,
    "ts": 1514656753908.213, "dur": 6.069,
    "args": {
      "External id": 2483434,"Record function id": 0, "Concrete Inputs": ["", "2", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], []], "Ev Idx": 233
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656753912.926, "dur": 1.171,
    "args": {
      "External id": 2483435,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[8388608, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], []], "Ev Idx": 234
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656753915.077, "dur": 30.319,
    "args": {
      "External id": 2483436,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 235
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SelectBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656753958.966, "dur": 90.711,
    "args": {
      "External id": 2483437,"Record function id": 0, "Sequence number": 24740621, "Fwd thread id": 1, "Ev Idx": 236
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SelectBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656753960.276, "dur": 85.626,
    "args": {
      "External id": 2483438,"Sequence number": 24740621, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 237
    }
  },
  {
    "ph": "f", "id": 27, "pid": 1336754, "tid": 1381179, "ts": 1514656753960.276,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select_backward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656753961.909, "dur": 83.702,
    "args": {
      "External id": 2483439,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "2", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], []], "Ev Idx": 238
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336754, "tid": 1381179,
    "ts": 1514656753963.623, "dur": 51.902,
    "args": {
      "External id": 2483440,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 239
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656753970.476, "dur": 2.568,
    "args": {
      "External id": 2483441,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 240
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656753973.647, "dur": 41.624,
    "args": {
      "External id": 2483442,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 241
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656753974.296, "dur": 40.148,
    "args": {
      "External id": 2483443,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], []], "Ev Idx": 242
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754017.260, "dur": 4.552,
    "args": {
      "External id": 2483444,"Record function id": 0, "Concrete Inputs": ["", "2", "0"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], []], "Input Dims": [[16, 4096, 4, 2048], [], []], "Ev Idx": 243
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754020.455, "dur": 0.855,
    "args": {
      "External id": 2483445,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 244
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754022.409, "dur": 22.544,
    "args": {
      "External id": 2483446,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 245
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SliceBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754055.366, "dur": 92.712,
    "args": {
      "External id": 2483447,"Record function id": 0, "Sequence number": 24740620, "Fwd thread id": 1, "Ev Idx": 246
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SliceBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754056.935, "dur": 88.188,
    "args": {
      "External id": 2483448,"Sequence number": 24740620, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 247
    }
  },
  {
    "ph": "f", "id": 28, "pid": 1336754, "tid": 1381179, "ts": 1514656754056.935,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice_backward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754058.867, "dur": 85.935,
    "args": {
      "External id": 2483449,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "1", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], [], []], "Ev Idx": 248
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754061.994, "dur": 19.261,
    "args": {
      "External id": 2483450,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 249
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754063.070, "dur": 2.390,
    "args": {
      "External id": 2483451,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 250
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754068.326, "dur": 12.674,
    "args": {
      "External id": 2483452,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 251
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754069.316, "dur": 11.398,
    "args": {
      "External id": 2483453,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], []], "Ev Idx": 252
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754082.245, "dur": 4.403,
    "args": {
      "External id": 2483454,"Record function id": 0, "Concrete Inputs": ["", "1", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 253
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754083.596, "dur": 2.826,
    "args": {
      "External id": 2483455,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 254
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754087.171, "dur": 57.024,
    "args": {
      "External id": 2483456,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], [16, 4096, 4, 2048], []], "Ev Idx": 255
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SliceBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754151.999, "dur": 106.192,
    "args": {
      "External id": 2483457,"Record function id": 0, "Sequence number": 24740619, "Fwd thread id": 1, "Ev Idx": 256
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SliceBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754153.263, "dur": 85.140,
    "args": {
      "External id": 2483458,"Sequence number": 24740619, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 257
    }
  },
  {
    "ph": "f", "id": 29, "pid": 1336754, "tid": 1381179, "ts": 1514656754153.263,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice_backward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754155.045, "dur": 83.036,
    "args": {
      "External id": 2483459,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "0", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], [], []], "Ev Idx": 258
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754156.257, "dur": 30.644,
    "args": {
      "External id": 2483460,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 259
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754161.839, "dur": 1.565,
    "args": {
      "External id": 2483461,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 260
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754168.171, "dur": 18.471,
    "args": {
      "External id": 2483462,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 261
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754173.661, "dur": 12.576,
    "args": {
      "External id": 2483463,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], []], "Ev Idx": 262
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754187.854, "dur": 2.066,
    "args": {
      "External id": 2483464,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 263
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754189.070, "dur": 0.711,
    "args": {
      "External id": 2483465,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 264
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754190.528, "dur": 46.886,
    "args": {
      "External id": 2483466,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], [16, 4096, 4, 2048], []], "Ev Idx": 265
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754242.143, "dur": 13.930,
    "args": {
      "External id": 2483467,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], [16, 4096, 4, 2048], []], "Ev Idx": 266
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754264.265, "dur": 427.240,
    "args": {
      "External id": 2483468,"Record function id": 0, "Sequence number": 24740618, "Fwd thread id": 1, "Ev Idx": 267
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754265.727, "dur": 413.580,
    "args": {
      "External id": 2483469,"Sequence number": 24740618, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 268
    }
  },
  {
    "ph": "f", "id": 30, "pid": 1336754, "tid": 1381179, "ts": 1514656754265.727,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754468.737, "dur": 49.963,
    "args": {
      "External id": 2483470,"kernel_hash": "cf7hg2ixt4i43dnksudd5khnoxtnhj2eb5bah2yjl2diidj4xwne", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "262144", "2048", "1", "1986", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/f7/cf7hg2ixt4i43dnksudd5khnoxtnhj2eb5bah2yjl2diidj4xwne.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[262144, 2048], [2048], [262144, 2048], [262144, 2048], [132, 2048], [262144], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 269
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754578.207, "dur": 33.045,
    "args": {
      "External id": 2483471,"kernel_hash": "c3qhxfu5zrkzpf2jybafupdlrhzermxghmhj46kd4kjqpnskd7hf", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/3q/c3qhxfu5zrkzpf2jybafupdlrhzermxghmhj46kd4kjqpnskd7hf.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 270
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754629.426, "dur": 20.962,
    "args": {
      "External id": 2483472,"kernel_hash": "chmktjza3gzdueqxxvt7hlbkykb7cdfv7coarxbxb3ldjxl472md", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/hm/chmktjza3gzdueqxxvt7hlbkykb7cdfv7coarxbxb3ldjxl472md.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 271
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754702.335, "dur": 11.289,
    "args": {
      "External id": 2483473,"Record function id": 0, "Ev Idx": 272
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754704.418, "dur": 8.455,
    "args": {
      "External id": 2483474,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 273
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754707.709, "dur": 4.460,
    "args": {
      "External id": 2483475,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 274
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754709.087, "dur": 2.904,
    "args": {
      "External id": 2483476,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 275
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: StackBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754720.223, "dur": 37.568,
    "args": {
      "External id": 2483477,"Record function id": 0, "Sequence number": 24740617, "Fwd thread id": 1, "Ev Idx": 276
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "StackBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754721.194, "dur": 26.141,
    "args": {
      "External id": 2483478,"Sequence number": 24740617, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 277
    }
  },
  {
    "ph": "f", "id": 31, "pid": 1336754, "tid": 1381179, "ts": 1514656754721.194,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754723.207, "dur": 7.702,
    "args": {
      "External id": 2483479,"Record function id": 0, "Concrete Inputs": ["", "-2", "0"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], []], "Input Dims": [[16, 4096, 4, 2048], [], []], "Ev Idx": 278
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754728.136, "dur": 1.046,
    "args": {
      "External id": 2483480,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 279
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754731.505, "dur": 7.447,
    "args": {
      "External id": 2483481,"Record function id": 0, "Concrete Inputs": ["", "-2", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], []], "Input Dims": [[16, 4096, 4, 2048], [], []], "Ev Idx": 280
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754737.305, "dur": 0.661,
    "args": {
      "External id": 2483482,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "2048"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 281
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754739.346, "dur": 2.429,
    "args": {
      "External id": 2483483,"Record function id": 0, "Concrete Inputs": ["", "-2", "2"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], []], "Input Dims": [[16, 4096, 4, 2048], [], []], "Ev Idx": 282
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754740.354, "dur": 0.449,
    "args": {
      "External id": 2483484,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "4096"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 283
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754742.140, "dur": 4.466,
    "args": {
      "External id": 2483485,"Record function id": 0, "Concrete Inputs": ["", "-2", "3"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], []], "Input Dims": [[16, 4096, 4, 2048], [], []], "Ev Idx": 284
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754745.450, "dur": 0.478,
    "args": {
      "External id": 2483486,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "6144"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 285
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: AddBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754761.480, "dur": 8.414,
    "args": {
      "External id": 2483487,"Record function id": 0, "Sequence number": 24740616, "Fwd thread id": 1, "Ev Idx": 286
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "AddBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754762.378, "dur": 1.192,
    "args": {
      "External id": 2483488,"Sequence number": 24740616, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 287
    }
  },
  {
    "ph": "f", "id": 32, "pid": 1336754, "tid": 1381179, "ts": 1514656754762.378,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SwiGLULinearFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754774.377, "dur": 625.394,
    "args": {
      "External id": 2483489,"Record function id": 0, "Sequence number": 24740615, "Fwd thread id": 1, "Ev Idx": 288
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SwiGLULinearFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754775.488, "dur": 607.160,
    "args": {
      "External id": 2483490,"Sequence number": 24740615, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 289
    }
  },
  {
    "ph": "f", "id": 33, "pid": 1336754, "tid": 1381179, "ts": 1514656754775.488,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754825.589, "dur": 12.002,
    "args": {
      "External id": 2483491,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 290
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_reshape_alias", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754833.093, "dur": 4.193,
    "args": {
      "External id": 2483492,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]", "[8192, 1]"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList"], "Input Strides": [[33554432, 8192, 1], [], []], "Input Dims": [[16, 4096, 2048], [], []], "Ev Idx": 291
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754844.577, "dur": 6.555,
    "args": {
      "External id": 2483493,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 292
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754847.733, "dur": 2.378,
    "args": {
      "External id": 2483494,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 293
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754849.031, "dur": 0.787,
    "args": {
      "External id": 2483495,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 294
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754859.920, "dur": 183.866,
    "args": {
      "External id": 2483496,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8192, 1], [1, 5632], []], "Input Dims": [[65536, 2048], [5632, 2048], []], "Ev Idx": 295
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754861.269, "dur": 7.433,
    "args": {
      "External id": 2483497,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 5632]], "Input Dims": [[5632, 2048]], "Ev Idx": 296
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754861.965, "dur": 5.935,
    "args": {
      "External id": 2483498,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 5632], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 297
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754865.229, "dur": 2.521,
    "args": {
      "External id": 2483499,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[5632, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 5632], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 298
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754870.775, "dur": 172.188,
    "args": {
      "External id": 2483500,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8192, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 299
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656754873.023, "dur": 168.389,
    "args": {
      "External id": 2483501,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8192, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 300
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755052.862, "dur": 7.283,
    "args": {
      "External id": 2483502,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [23068672, 5632, 1]], "Input Dims": [[65536, 5632], [16, 4096, 5632]], "Ev Idx": 301
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755056.856, "dur": 3.166,
    "args": {
      "External id": 2483503,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 302
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755103.861, "dur": 5.769,
    "args": {
      "External id": 2483504,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 303
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755110.596, "dur": 4.544,
    "args": {
      "External id": 2483505,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 304
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755118.215, "dur": 2.275,
    "args": {
      "External id": 2483506,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 305
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755163.291, "dur": 2.379,
    "args": {
      "External id": 2483507,"Record function id": 0, "Concrete Inputs": ["", "[-1, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 306
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755164.137, "dur": 1.367,
    "args": {
      "External id": 2483508,"Record function id": 0, "Concrete Inputs": ["", "[-1, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 307
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::einsum", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755195.822, "dur": 164.117,
    "args": {
      "External id": 2483509,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["", "TensorList", ""], "Input Strides": [[], [[8192, 1], [5632, 1]], []], "Input Dims": [[], [[65536, 2048], [65536, 5632]], []], "Ev Idx": 308
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755203.481, "dur": 9.437,
    "args": {
      "External id": 2483510,"Record function id": 0, "Concrete Inputs": ["", "2"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 309
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755209.066, "dur": 1.270,
    "args": {
      "External id": 2483511,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048, 1]", "[8192, 1, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[8192, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 310
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755215.960, "dur": 9.802,
    "args": {
      "External id": 2483512,"Record function id": 0, "Concrete Inputs": ["", "[1, 2, 0]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8192, 1, 1], []], "Input Dims": [[65536, 2048, 1], []], "Ev Idx": 311
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755221.448, "dur": 3.529,
    "args": {
      "External id": 2483513,"Record function id": 0, "Concrete Inputs": ["", "[2048, 1, 65536]", "[1, 1, 8192]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[65536, 2048, 1], [], [], []], "Ev Idx": 312
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755227.332, "dur": 4.123,
    "args": {
      "External id": 2483514,"Record function id": 0, "Concrete Inputs": ["", "2"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 313
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755230.657, "dur": 0.450,
    "args": {
      "External id": 2483515,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632, 1]", "[5632, 1, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[65536, 5632], [], [], []], "Ev Idx": 314
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755232.377, "dur": 4.717,
    "args": {
      "External id": 2483516,"Record function id": 0, "Concrete Inputs": ["", "[2, 1, 0]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 1], []], "Input Dims": [[65536, 5632, 1], []], "Ev Idx": 315
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755235.891, "dur": 0.468,
    "args": {
      "External id": 2483517,"Record function id": 0, "Concrete Inputs": ["", "[1, 5632, 65536]", "[1, 1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1, 1], [], [], []], "Input Dims": [[65536, 5632, 1], [], [], []], "Ev Idx": 316
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755241.095, "dur": 4.226,
    "args": {
      "External id": 2483518,"Record function id": 0, "Concrete Inputs": ["", "[0, 2, 1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1, 1, 8192], []], "Input Dims": [[2048, 1, 65536], []], "Ev Idx": 317
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755244.213, "dur": 0.798,
    "args": {
      "External id": 2483519,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536, 1]", "[1, 8192, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1, 8192], [], [], []], "Input Dims": [[2048, 1, 65536], [], [], []], "Ev Idx": 318
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755248.204, "dur": 7.517,
    "args": {
      "External id": 2483520,"Record function id": 0, "Concrete Inputs": ["", "[1, 2048, 65536]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1, 8192, 1], []], "Input Dims": [[2048, 65536, 1], []], "Ev Idx": 319
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_reshape_alias", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755253.531, "dur": 1.982,
    "args": {
      "External id": 2483521,"Record function id": 0, "Concrete Inputs": ["", "[1, 2048, 65536]", "[2048, 1, 8192]"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList"], "Input Strides": [[1, 8192, 1], [], []], "Input Dims": [[2048, 65536, 1], [], []], "Ev Idx": 320
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755256.447, "dur": 3.681,
    "args": {
      "External id": 2483522,"Record function id": 0, "Concrete Inputs": ["", "[2, 1, 0]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1, 1, 5632], []], "Input Dims": [[1, 5632, 65536], []], "Ev Idx": 321
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755259.464, "dur": 0.314,
    "args": {
      "External id": 2483523,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632, 1]", "[5632, 1, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1, 5632], [], [], []], "Input Dims": [[1, 5632, 65536], [], [], []], "Ev Idx": 322
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755260.871, "dur": 2.060,
    "args": {
      "External id": 2483524,"Record function id": 0, "Concrete Inputs": ["", "[1, 65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 1], []], "Input Dims": [[65536, 5632, 1], []], "Ev Idx": 323
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755261.612, "dur": 1.225,
    "args": {
      "External id": 2483525,"Record function id": 0, "Concrete Inputs": ["", "[1, 65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 1], []], "Input Dims": [[65536, 5632, 1], []], "Ev Idx": 324
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755265.119, "dur": 79.912,
    "args": {
      "External id": 2483526,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1, 8192], [369098752, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632]], "Ev Idx": 325
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755346.747, "dur": 3.052,
    "args": {
      "External id": 2483527,"Record function id": 0, "Concrete Inputs": ["", "[2048, 1, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[11534336, 5632, 1], []], "Input Dims": [[1, 2048, 5632], []], "Ev Idx": 326
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755352.703, "dur": 2.343,
    "args": {
      "External id": 2483528,"Record function id": 0, "Concrete Inputs": ["", "[0, 2, 1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 5632, 1], []], "Input Dims": [[2048, 1, 5632], []], "Ev Idx": 327
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755353.779, "dur": 0.693,
    "args": {
      "External id": 2483529,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632, 1]", "[5632, 1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 5632, 1], [], [], []], "Input Dims": [[2048, 1, 5632], [], [], []], "Ev Idx": 328
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755357.022, "dur": 1.465,
    "args": {
      "External id": 2483530,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 5632], []], "Input Dims": [[2048, 5632, 1], []], "Ev Idx": 329
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755412.137, "dur": 9.712,
    "args": {
      "External id": 2483531,"Record function id": 0, "Ev Idx": 330
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755414.350, "dur": 6.711,
    "args": {
      "External id": 2483532,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 331
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755416.725, "dur": 3.555,
    "args": {
      "External id": 2483533,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 332
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755417.836, "dur": 2.350,
    "args": {
      "External id": 2483534,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 333
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755426.111, "dur": 8.990,
    "args": {
      "External id": 2483535,"Record function id": 0, "Sequence number": 24740614, "Fwd thread id": 1, "Ev Idx": 334
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755427.117, "dur": 5.391,
    "args": {
      "External id": 2483536,"Sequence number": 24740614, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[23068672, 5632, 1]], "Input Dims": [[16, 4096, 5632]], "Ev Idx": 335
    }
  },
  {
    "ph": "f", "id": 34, "pid": 1336754, "tid": 1381179, "ts": 1514656755427.117,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755430.352, "dur": 1.943,
    "args": {
      "External id": 2483537,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 336
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755431.193, "dur": 0.979,
    "args": {
      "External id": 2483538,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 337
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755438.938, "dur": 195.577,
    "args": {
      "External id": 2483539,"Record function id": 0, "Sequence number": 24740613, "Fwd thread id": 1, "Ev Idx": 338
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755439.889, "dur": 184.492,
    "args": {
      "External id": 2483540,"Sequence number": 24740613, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 339
    }
  },
  {
    "ph": "f", "id": 35, "pid": 1336754, "tid": 1381179, "ts": 1514656755439.889,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755448.398, "dur": 3.920,
    "args": {
      "External id": 2483541,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 340
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755449.797, "dur": 1.930,
    "args": {
      "External id": 2483542,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[65536, 5632], [], []], "Ev Idx": 341
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755450.979, "dur": 0.586,
    "args": {
      "External id": 2483543,"Record function id": 0, "Concrete Inputs": ["", "[5632, 65536]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[65536, 5632], [], [], []], "Ev Idx": 342
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755453.566, "dur": 56.840,
    "args": {
      "External id": 2483544,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048]], "Ev Idx": 343
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755511.540, "dur": 5.834,
    "args": {
      "External id": 2483545,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 344
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755512.289, "dur": 4.568,
    "args": {
      "External id": 2483546,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 345
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755515.749, "dur": 0.935,
    "args": {
      "External id": 2483547,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 346
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755518.984, "dur": 6.180,
    "args": {
      "External id": 2483548,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 347
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755520.271, "dur": 4.153,
    "args": {
      "External id": 2483549,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 348
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755523.626, "dur": 0.732,
    "args": {
      "External id": 2483550,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 349
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755525.664, "dur": 97.392,
    "args": {
      "External id": 2483551,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048]], "Ev Idx": 350
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755641.892, "dur": 7.674,
    "args": {
      "External id": 2483552,"Record function id": 0, "Sequence number": 24740612, "Fwd thread id": 1, "Ev Idx": 351
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755643.119, "dur": 4.458,
    "args": {
      "External id": 2483553,"Sequence number": 24740612, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 352
    }
  },
  {
    "ph": "f", "id": 36, "pid": 1336754, "tid": 1381179, "ts": 1514656755643.119,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755644.906, "dur": 2.488,
    "args": {
      "External id": 2483554,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 353
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755645.652, "dur": 1.596,
    "args": {
      "External id": 2483555,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 354
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755653.284, "dur": 13.032,
    "args": {
      "External id": 2483556,"Record function id": 0, "Sequence number": 24740611, "Fwd thread id": 1, "Ev Idx": 355
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755656.432, "dur": 6.840,
    "args": {
      "External id": 2483557,"Sequence number": 24740611, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 356
    }
  },
  {
    "ph": "f", "id": 37, "pid": 1336754, "tid": 1381179, "ts": 1514656755656.432,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755657.611, "dur": 5.436,
    "args": {
      "External id": 2483558,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 357
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755658.763, "dur": 3.789,
    "args": {
      "External id": 2483559,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 358
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755661.641, "dur": 0.795,
    "args": {
      "External id": 2483560,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 359
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755670.267, "dur": 5.914,
    "args": {
      "External id": 2483561,"Record function id": 0, "Ev Idx": 360
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755671.692, "dur": 3.972,
    "args": {
      "External id": 2483562,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 361
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755673.230, "dur": 2.058,
    "args": {
      "External id": 2483563,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 362
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755673.762, "dur": 1.413,
    "args": {
      "External id": 2483564,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 363
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755679.310, "dur": 8.760,
    "args": {
      "External id": 2483565,"Record function id": 0, "Sequence number": 24740610, "Fwd thread id": 1, "Ev Idx": 364
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755680.223, "dur": 4.426,
    "args": {
      "External id": 2483566,"Sequence number": 24740610, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[23068672, 5632, 1]], "Input Dims": [[16, 4096, 5632]], "Ev Idx": 365
    }
  },
  {
    "ph": "f", "id": 38, "pid": 1336754, "tid": 1381179, "ts": 1514656755680.223,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755681.527, "dur": 2.982,
    "args": {
      "External id": 2483567,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 366
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755683.761, "dur": 0.628,
    "args": {
      "External id": 2483568,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 367
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755691.418, "dur": 125.551,
    "args": {
      "External id": 2483569,"Record function id": 0, "Sequence number": 24740609, "Fwd thread id": 1, "Ev Idx": 368
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755692.299, "dur": 113.541,
    "args": {
      "External id": 2483570,"Sequence number": 24740609, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 369
    }
  },
  {
    "ph": "f", "id": 39, "pid": 1336754, "tid": 1381179, "ts": 1514656755692.299,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755695.948, "dur": 2.571,
    "args": {
      "External id": 2483571,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 370
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755696.376, "dur": 1.746,
    "args": {
      "External id": 2483572,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[65536, 5632], [], []], "Ev Idx": 371
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755697.409, "dur": 0.600,
    "args": {
      "External id": 2483573,"Record function id": 0, "Concrete Inputs": ["", "[5632, 65536]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[65536, 5632], [], [], []], "Ev Idx": 372
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755701.620, "dur": 36.890,
    "args": {
      "External id": 2483574,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048]], "Ev Idx": 373
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755739.530, "dur": 6.010,
    "args": {
      "External id": 2483575,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 374
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755740.321, "dur": 4.743,
    "args": {
      "External id": 2483576,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 375
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755743.719, "dur": 1.212,
    "args": {
      "External id": 2483577,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 376
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755747.052, "dur": 8.245,
    "args": {
      "External id": 2483578,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 377
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755747.947, "dur": 6.691,
    "args": {
      "External id": 2483579,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 378
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755751.077, "dur": 3.495,
    "args": {
      "External id": 2483580,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 379
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755755.854, "dur": 49.316,
    "args": {
      "External id": 2483581,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048]], "Ev Idx": 380
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755824.262, "dur": 38.912,
    "args": {
      "External id": 2483582,"Record function id": 0, "Sequence number": 24740608, "Fwd thread id": 1, "Ev Idx": 381
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755825.366, "dur": 6.636,
    "args": {
      "External id": 2483583,"Sequence number": 24740608, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 382
    }
  },
  {
    "ph": "f", "id": 40, "pid": 1336754, "tid": 1381179, "ts": 1514656755825.366,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755827.143, "dur": 4.689,
    "args": {
      "External id": 2483584,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 383
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755830.352, "dur": 1.378,
    "args": {
      "External id": 2483585,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 384
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755836.022, "dur": 24.012,
    "args": {
      "External id": 2483586,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 385
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755867.011, "dur": 10.276,
    "args": {
      "External id": 2483587,"Record function id": 0, "Sequence number": 24740607, "Fwd thread id": 1, "Ev Idx": 386
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755868.079, "dur": 6.385,
    "args": {
      "External id": 2483588,"Sequence number": 24740607, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 387
    }
  },
  {
    "ph": "f", "id": 41, "pid": 1336754, "tid": 1381179, "ts": 1514656755868.079,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755869.064, "dur": 5.194,
    "args": {
      "External id": 2483589,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 388
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755872.007, "dur": 1.773,
    "args": {
      "External id": 2483590,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 389
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755872.983, "dur": 0.631,
    "args": {
      "External id": 2483591,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 390
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755881.420, "dur": 5.587,
    "args": {
      "External id": 2483592,"Record function id": 0, "Ev Idx": 391
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755882.887, "dur": 3.604,
    "args": {
      "External id": 2483593,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 392
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755884.282, "dur": 1.938,
    "args": {
      "External id": 2483594,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 393
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755884.855, "dur": 1.289,
    "args": {
      "External id": 2483595,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 394
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: LayerNormFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755891.654, "dur": 552.706,
    "args": {
      "External id": 2483596,"Record function id": 0, "Sequence number": 24740606, "Fwd thread id": 1, "Ev Idx": 395
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755893.045, "dur": 530.158,
    "args": {
      "External id": 2483597,"Sequence number": 24740606, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [33554432, 8192, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 396
    }
  },
  {
    "ph": "f", "id": 42, "pid": 1336754, "tid": 1381179, "ts": 1514656755893.045,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::contiguous", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755921.605, "dur": 37.467,
    "args": {
      "External id": 2483598,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 397
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755923.931, "dur": 34.942,
    "args": {
      "External id": 2483599,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 398
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755927.642, "dur": 6.113,
    "args": {
      "External id": 2483600,"Record function id": 0, "Concrete Inputs": ["", "15", "0", "", "", "0"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[33554432, 8192, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], [], []], "Ev Idx": 399
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755930.194, "dur": 2.902,
    "args": {
      "External id": 2483601,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 2048]", "15", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 400
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755935.023, "dur": 23.140,
    "args": {
      "External id": 2483602,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 401
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755973.252, "dur": 3.849,
    "args": {
      "External id": 2483603,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 402
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656755975.870, "dur": 1.068,
    "args": {
      "External id": 2483604,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 403
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756015.574, "dur": 5.608,
    "args": {
      "External id": 2483605,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 404
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756017.247, "dur": 3.672,
    "args": {
      "External id": 2483606,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 405
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756041.812, "dur": 4.035,
    "args": {
      "External id": 2483607,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 406
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756062.694, "dur": 3.831,
    "args": {
      "External id": 2483608,"Record function id": 0, "Concrete Inputs": ["[132, 2048]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 407
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756304.679, "dur": 2.410,
    "args": {
      "External id": 2483609,"Record function id": 0, "Concrete Inputs": ["", "[1, -1, 2048]"], "Input type": ["float", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[132, 2048], []], "Ev Idx": 408
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756312.493, "dur": 38.152,
    "args": {
      "External id": 2483610,"Record function id": 0, "Concrete Inputs": ["", "[1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[270336, 2048, 1], [], [], []], "Input Dims": [[1, 132, 2048], [], [], []], "Ev Idx": 409
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756324.284, "dur": 0.968,
    "args": {
      "External id": 2483611,"Record function id": 0, "Concrete Inputs": ["", "[1, 1, 2048]", "[2048, 0, 1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[1, 2048], [], [], []], "Ev Idx": 410
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756357.230, "dur": 29.800,
    "args": {
      "External id": 2483612,"Record function id": 0, "Concrete Inputs": ["", "", "15", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], []], "Ev Idx": 411
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756358.927, "dur": 27.912,
    "args": {
      "External id": 2483613,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], [], []], "Ev Idx": 412
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756363.039, "dur": 4.125,
    "args": {
      "External id": 2483614,"Record function id": 0, "Concrete Inputs": ["[1, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 413
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756368.321, "dur": 18.038,
    "args": {
      "External id": 2483615,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "float", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[1, 2048], [1, 2048], []], "Ev Idx": 414
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756393.944, "dur": 2.545,
    "args": {
      "External id": 2483616,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1]], "Input Dims": [[1, 2048], [2048]], "Ev Idx": 415
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756395.025, "dur": 1.317,
    "args": {
      "External id": 2483617,"Record function id": 0, "Concrete Inputs": ["", "[2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[1, 2048], []], "Ev Idx": 416
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756403.963, "dur": 4.642,
    "args": {
      "External id": 2483618,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 417
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756407.161, "dur": 1.318,
    "args": {
      "External id": 2483619,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 418
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756410.631, "dur": 3.403,
    "args": {
      "External id": 2483620,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 419
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756411.291, "dur": 2.652,
    "args": {
      "External id": 2483621,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 420
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756458.050, "dur": 9.599,
    "args": {
      "External id": 2483622,"Record function id": 0, "Ev Idx": 421
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756460.785, "dur": 6.186,
    "args": {
      "External id": 2483623,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 422
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756463.046, "dur": 3.069,
    "args": {
      "External id": 2483624,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 423
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756464.030, "dur": 2.005,
    "args": {
      "External id": 2483625,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 424
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756471.587, "dur": 10.415,
    "args": {
      "External id": 2483626,"Record function id": 0, "Sequence number": 24740605, "Fwd thread id": 1, "Ev Idx": 425
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756472.956, "dur": 5.768,
    "args": {
      "External id": 2483627,"Sequence number": 24740605, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 426
    }
  },
  {
    "ph": "f", "id": 43, "pid": 1336754, "tid": 1381179, "ts": 1514656756472.956,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756476.328, "dur": 2.172,
    "args": {
      "External id": 2483628,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 427
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756477.288, "dur": 1.053,
    "args": {
      "External id": 2483629,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 428
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756488.135, "dur": 172.865,
    "args": {
      "External id": 2483630,"Record function id": 0, "Sequence number": 24740604, "Fwd thread id": 1, "Ev Idx": 429
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756489.151, "dur": 166.504,
    "args": {
      "External id": 2483631,"Sequence number": 24740604, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 430
    }
  },
  {
    "ph": "f", "id": 44, "pid": 1336754, "tid": 1381179, "ts": 1514656756489.151,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756493.920, "dur": 4.263,
    "args": {
      "External id": 2483632,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 431
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756495.312, "dur": 2.197,
    "args": {
      "External id": 2483633,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 432
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756496.593, "dur": 0.696,
    "args": {
      "External id": 2483634,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 433
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756499.306, "dur": 85.180,
    "args": {
      "External id": 2483635,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 434
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756586.744, "dur": 7.988,
    "args": {
      "External id": 2483636,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 435
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756589.788, "dur": 4.207,
    "args": {
      "External id": 2483637,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 436
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756591.087, "dur": 2.726,
    "args": {
      "External id": 2483638,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 437
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756596.422, "dur": 5.425,
    "args": {
      "External id": 2483639,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 438
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756597.752, "dur": 3.677,
    "args": {
      "External id": 2483640,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 439
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756600.854, "dur": 0.493,
    "args": {
      "External id": 2483641,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 440
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756602.509, "dur": 52.279,
    "args": {
      "External id": 2483642,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 441
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756667.810, "dur": 14.159,
    "args": {
      "External id": 2483643,"Record function id": 0, "Sequence number": 24740603, "Fwd thread id": 1, "Ev Idx": 442
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756669.207, "dur": 9.408,
    "args": {
      "External id": 2483644,"Sequence number": 24740603, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 443
    }
  },
  {
    "ph": "f", "id": 45, "pid": 1336754, "tid": 1381179, "ts": 1514656756669.207,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756673.647, "dur": 4.792,
    "args": {
      "External id": 2483645,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 444
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756674.532, "dur": 3.810,
    "args": {
      "External id": 2483646,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 445
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756685.501, "dur": 10.261,
    "args": {
      "External id": 2483647,"Record function id": 0, "Sequence number": 24740602, "Fwd thread id": 1, "Ev Idx": 446
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756686.478, "dur": 7.102,
    "args": {
      "External id": 2483648,"Sequence number": 24740602, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 447
    }
  },
  {
    "ph": "f", "id": 46, "pid": 1336754, "tid": 1381179, "ts": 1514656756686.478,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756687.844, "dur": 5.510,
    "args": {
      "External id": 2483649,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 448
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756688.610, "dur": 4.247,
    "args": {
      "External id": 2483650,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 449
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756692.077, "dur": 0.701,
    "args": {
      "External id": 2483651,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 450
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756700.076, "dur": 7.990,
    "args": {
      "External id": 2483652,"Record function id": 0, "Ev Idx": 451
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756701.313, "dur": 6.226,
    "args": {
      "External id": 2483653,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 452
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756702.674, "dur": 4.559,
    "args": {
      "External id": 2483654,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 453
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756705.500, "dur": 1.624,
    "args": {
      "External id": 2483655,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 454
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756711.425, "dur": 7.377,
    "args": {
      "External id": 2483656,"Record function id": 0, "Sequence number": 24740601, "Fwd thread id": 1, "Ev Idx": 455
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756712.501, "dur": 3.624,
    "args": {
      "External id": 2483657,"Sequence number": 24740601, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 456
    }
  },
  {
    "ph": "f", "id": 47, "pid": 1336754, "tid": 1381179, "ts": 1514656756712.501,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756713.899, "dur": 2.094,
    "args": {
      "External id": 2483658,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 457
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756714.677, "dur": 1.171,
    "args": {
      "External id": 2483659,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 458
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: FlashAttnFuncBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756723.714, "dur": 397.368,
    "args": {
      "External id": 2483660,"Record function id": 0, "Sequence number": 24740600, "Fwd thread id": 1, "Ev Idx": 459
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FlashAttnFuncBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756725.147, "dur": 368.352,
    "args": {
      "External id": 2483661,"Sequence number": 24740600, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 460
    }
  },
  {
    "ph": "f", "id": 48, "pid": 1336754, "tid": 1381179, "ts": 1514656756725.147,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756747.943, "dur": 10.271,
    "args": {
      "External id": 2483662,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 461
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756754.583, "dur": 3.203,
    "args": {
      "External id": 2483663,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 462
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756760.349, "dur": 3.113,
    "args": {
      "External id": 2483664,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 463
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756761.310, "dur": 1.921,
    "args": {
      "External id": 2483665,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 464
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756765.094, "dur": 6.004,
    "args": {
      "External id": 2483666,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 465
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756767.769, "dur": 3.101,
    "args": {
      "External id": 2483667,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 466
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756806.009, "dur": 256.009,
    "args": {
      "External id": 2483668,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 467
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756912.713, "dur": 4.102,
    "args": {
      "External id": 2483669,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 468
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656756918.786, "dur": 4.547,
    "args": {
      "External id": 2483670,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 469
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757079.335, "dur": 4.604,
    "args": {
      "External id": 2483671,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 470
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757086.940, "dur": 1.002,
    "args": {
      "External id": 2483672,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 471
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757089.773, "dur": 0.770,
    "args": {
      "External id": 2483673,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 472
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RotaryEmbeddingFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757131.491, "dur": 257.745,
    "args": {
      "External id": 2483674,"Record function id": 0, "Sequence number": 24740599, "Fwd thread id": 1, "Ev Idx": 473
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757133.148, "dur": 247.911,
    "args": {
      "External id": 2483675,"Sequence number": 24740599, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 474
    }
  },
  {
    "ph": "f", "id": 49, "pid": 1336754, "tid": 1381179, "ts": 1514656757133.148,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757157.459, "dur": 55.005,
    "args": {
      "External id": 2483676,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 475
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757164.513, "dur": 3.661,
    "args": {
      "External id": 2483677,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 476
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757169.477, "dur": 42.329,
    "args": {
      "External id": 2483678,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], []], "Ev Idx": 477
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757223.305, "dur": 3.912,
    "args": {
      "External id": 2483679,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 478
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757224.898, "dur": 1.980,
    "args": {
      "External id": 2483680,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 479
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RotaryEmbeddingFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757396.701, "dur": 177.649,
    "args": {
      "External id": 2483681,"Record function id": 0, "Sequence number": 24740598, "Fwd thread id": 1, "Ev Idx": 480
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757398.366, "dur": 167.461,
    "args": {
      "External id": 2483682,"Sequence number": 24740598, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 481
    }
  },
  {
    "ph": "f", "id": 50, "pid": 1336754, "tid": 1381179, "ts": 1514656757398.366,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757409.948, "dur": 33.095,
    "args": {
      "External id": 2483683,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 482
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757414.233, "dur": 2.484,
    "args": {
      "External id": 2483684,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 483
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757417.553, "dur": 25.017,
    "args": {
      "External id": 2483685,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], []], "Ev Idx": 484
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757449.643, "dur": 3.950,
    "args": {
      "External id": 2483686,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 485
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757450.990, "dur": 2.284,
    "args": {
      "External id": 2483687,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 486
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757585.310, "dur": 16.925,
    "args": {
      "External id": 2483688,"Record function id": 0, "Sequence number": 24740597, "Fwd thread id": 1, "Ev Idx": 487
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757587.164, "dur": 10.960,
    "args": {
      "External id": 2483689,"Sequence number": 24740597, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 488
    }
  },
  {
    "ph": "f", "id": 51, "pid": 1336754, "tid": 1381179, "ts": 1514656757587.164,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757589.832, "dur": 8.044,
    "args": {
      "External id": 2483690,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 489
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757591.013, "dur": 6.661,
    "args": {
      "External id": 2483691,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 490
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757605.609, "dur": 10.106,
    "args": {
      "External id": 2483692,"Record function id": 0, "Sequence number": 24740596, "Fwd thread id": 1, "Ev Idx": 491
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757606.951, "dur": 5.598,
    "args": {
      "External id": 2483693,"Sequence number": 24740596, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 492
    }
  },
  {
    "ph": "f", "id": 52, "pid": 1336754, "tid": 1381179, "ts": 1514656757606.951,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757610.576, "dur": 1.831,
    "args": {
      "External id": 2483694,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 493
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757611.493, "dur": 0.766,
    "args": {
      "External id": 2483695,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 494
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757618.891, "dur": 8.789,
    "args": {
      "External id": 2483696,"Record function id": 0, "Sequence number": 24740595, "Fwd thread id": 1, "Ev Idx": 495
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757620.007, "dur": 5.061,
    "args": {
      "External id": 2483697,"Sequence number": 24740595, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 496
    }
  },
  {
    "ph": "f", "id": 53, "pid": 1336754, "tid": 1381179, "ts": 1514656757620.007,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757623.284, "dur": 1.644,
    "args": {
      "External id": 2483698,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 497
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757623.828, "dur": 1.005,
    "args": {
      "External id": 2483699,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 498
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757633.111, "dur": 7.049,
    "args": {
      "External id": 2483700,"Record function id": 0, "Sequence number": 24740594, "Fwd thread id": 1, "Ev Idx": 499
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757634.095, "dur": 3.258,
    "args": {
      "External id": 2483701,"Sequence number": 24740594, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 500
    }
  },
  {
    "ph": "f", "id": 54, "pid": 1336754, "tid": 1381179, "ts": 1514656757634.095,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757635.594, "dur": 1.613,
    "args": {
      "External id": 2483702,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 501
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757636.104, "dur": 1.007,
    "args": {
      "External id": 2483703,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 502
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757643.651, "dur": 187.891,
    "args": {
      "External id": 2483704,"Record function id": 0, "Sequence number": 24740593, "Fwd thread id": 1, "Ev Idx": 503
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757644.709, "dur": 178.016,
    "args": {
      "External id": 2483705,"Sequence number": 24740593, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 504
    }
  },
  {
    "ph": "f", "id": 55, "pid": 1336754, "tid": 1381179, "ts": 1514656757644.709,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757649.945, "dur": 8.288,
    "args": {
      "External id": 2483706,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 505
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757654.052, "dur": 3.541,
    "args": {
      "External id": 2483707,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 506
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757656.027, "dur": 1.299,
    "args": {
      "External id": 2483708,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 507
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757659.864, "dur": 70.836,
    "args": {
      "External id": 2483709,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 508
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757732.068, "dur": 6.618,
    "args": {
      "External id": 2483710,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 509
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757732.718, "dur": 5.052,
    "args": {
      "External id": 2483711,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 510
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757736.506, "dur": 1.130,
    "args": {
      "External id": 2483712,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 511
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757766.264, "dur": 5.559,
    "args": {
      "External id": 2483713,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 512
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757770.141, "dur": 1.221,
    "args": {
      "External id": 2483714,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 513
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757770.786, "dur": 0.508,
    "args": {
      "External id": 2483715,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 514
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757772.534, "dur": 49.333,
    "args": {
      "External id": 2483716,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 515
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757836.768, "dur": 6.566,
    "args": {
      "External id": 2483717,"Record function id": 0, "Sequence number": 24740592, "Fwd thread id": 1, "Ev Idx": 516
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757837.672, "dur": 3.645,
    "args": {
      "External id": 2483718,"Sequence number": 24740592, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 517
    }
  },
  {
    "ph": "f", "id": 56, "pid": 1336754, "tid": 1381179, "ts": 1514656757837.672,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757839.177, "dur": 1.985,
    "args": {
      "External id": 2483719,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 518
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757839.698, "dur": 1.371,
    "args": {
      "External id": 2483720,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 519
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757846.680, "dur": 11.271,
    "args": {
      "External id": 2483721,"Record function id": 0, "Sequence number": 24740591, "Fwd thread id": 1, "Ev Idx": 520
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757849.817, "dur": 5.473,
    "args": {
      "External id": 2483722,"Sequence number": 24740591, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 521
    }
  },
  {
    "ph": "f", "id": 57, "pid": 1336754, "tid": 1381179, "ts": 1514656757849.817,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757850.650, "dur": 4.424,
    "args": {
      "External id": 2483723,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 522
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757851.276, "dur": 3.327,
    "args": {
      "External id": 2483724,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 523
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757854.110, "dur": 0.384,
    "args": {
      "External id": 2483725,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 524
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757863.855, "dur": 8.792,
    "args": {
      "External id": 2483726,"Record function id": 0, "Ev Idx": 525
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757865.327, "dur": 6.543,
    "args": {
      "External id": 2483727,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 526
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757867.683, "dur": 3.712,
    "args": {
      "External id": 2483728,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 527
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757868.800, "dur": 2.483,
    "args": {
      "External id": 2483729,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 528
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757877.944, "dur": 8.557,
    "args": {
      "External id": 2483730,"Record function id": 0, "Sequence number": 24740590, "Fwd thread id": 1, "Ev Idx": 529
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757878.892, "dur": 5.323,
    "args": {
      "External id": 2483731,"Sequence number": 24740590, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 530
    }
  },
  {
    "ph": "f", "id": 58, "pid": 1336754, "tid": 1381179, "ts": 1514656757878.892,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757880.269, "dur": 3.804,
    "args": {
      "External id": 2483732,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 531
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757883.234, "dur": 0.725,
    "args": {
      "External id": 2483733,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 532
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757889.900, "dur": 133.430,
    "args": {
      "External id": 2483734,"Record function id": 0, "Sequence number": 24740589, "Fwd thread id": 1, "Ev Idx": 533
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757890.659, "dur": 88.203,
    "args": {
      "External id": 2483735,"Sequence number": 24740589, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 534
    }
  },
  {
    "ph": "f", "id": 59, "pid": 1336754, "tid": 1381179, "ts": 1514656757890.659,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757893.374, "dur": 4.722,
    "args": {
      "External id": 2483736,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 535
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757893.960, "dur": 3.716,
    "args": {
      "External id": 2483737,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 536
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757897.044, "dur": 0.521,
    "args": {
      "External id": 2483738,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 537
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757900.967, "dur": 30.359,
    "args": {
      "External id": 2483739,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 538
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757932.278, "dur": 2.624,
    "args": {
      "External id": 2483740,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 539
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757932.729, "dur": 1.633,
    "args": {
      "External id": 2483741,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 540
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757933.525, "dur": 0.713,
    "args": {
      "External id": 2483742,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 541
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757936.141, "dur": 6.966,
    "args": {
      "External id": 2483743,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 542
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757937.114, "dur": 5.430,
    "args": {
      "External id": 2483744,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 543
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757940.082, "dur": 2.390,
    "args": {
      "External id": 2483745,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 544
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656757943.571, "dur": 34.617,
    "args": {
      "External id": 2483746,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 545
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758031.124, "dur": 38.709,
    "args": {
      "External id": 2483747,"Record function id": 0, "Sequence number": 24740588, "Fwd thread id": 1, "Ev Idx": 546
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758032.277, "dur": 6.289,
    "args": {
      "External id": 2483748,"Sequence number": 24740588, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 547
    }
  },
  {
    "ph": "f", "id": 60, "pid": 1336754, "tid": 1381179, "ts": 1514656758032.277,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758036.172, "dur": 2.209,
    "args": {
      "External id": 2483749,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 548
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758036.646, "dur": 1.635,
    "args": {
      "External id": 2483750,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 549
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758041.633, "dur": 25.580,
    "args": {
      "External id": 2483751,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 550
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758073.619, "dur": 9.205,
    "args": {
      "External id": 2483752,"Record function id": 0, "Sequence number": 24740587, "Fwd thread id": 1, "Ev Idx": 551
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758074.718, "dur": 6.188,
    "args": {
      "External id": 2483753,"Sequence number": 24740587, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 552
    }
  },
  {
    "ph": "f", "id": 61, "pid": 1336754, "tid": 1381179, "ts": 1514656758074.718,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758075.673, "dur": 5.015,
    "args": {
      "External id": 2483754,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 553
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758076.337, "dur": 3.831,
    "args": {
      "External id": 2483755,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 554
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758079.495, "dur": 0.520,
    "args": {
      "External id": 2483756,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 555
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758087.108, "dur": 7.994,
    "args": {
      "External id": 2483757,"Record function id": 0, "Ev Idx": 556
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758090.659, "dur": 3.905,
    "args": {
      "External id": 2483758,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 557
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758091.925, "dur": 2.200,
    "args": {
      "External id": 2483759,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 558
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758092.452, "dur": 1.540,
    "args": {
      "External id": 2483760,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 559
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758098.331, "dur": 6.225,
    "args": {
      "External id": 2483761,"Record function id": 0, "Sequence number": 24740586, "Fwd thread id": 1, "Ev Idx": 560
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758099.278, "dur": 2.851,
    "args": {
      "External id": 2483762,"Sequence number": 24740586, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 561
    }
  },
  {
    "ph": "f", "id": 62, "pid": 1336754, "tid": 1381179, "ts": 1514656758099.278,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758100.487, "dur": 1.504,
    "args": {
      "External id": 2483763,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 562
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758100.965, "dur": 0.887,
    "args": {
      "External id": 2483764,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 563
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758107.705, "dur": 101.168,
    "args": {
      "External id": 2483765,"Record function id": 0, "Sequence number": 24740585, "Fwd thread id": 1, "Ev Idx": 564
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758110.505, "dur": 88.667,
    "args": {
      "External id": 2483766,"Sequence number": 24740585, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 565
    }
  },
  {
    "ph": "f", "id": 63, "pid": 1336754, "tid": 1381179, "ts": 1514656758110.505,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758113.588, "dur": 1.982,
    "args": {
      "External id": 2483767,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 566
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758114.004, "dur": 1.150,
    "args": {
      "External id": 2483768,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 567
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758114.614, "dur": 0.427,
    "args": {
      "External id": 2483769,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 568
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758116.342, "dur": 35.237,
    "args": {
      "External id": 2483770,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 569
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758152.483, "dur": 7.469,
    "args": {
      "External id": 2483771,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 570
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758153.052, "dur": 6.312,
    "args": {
      "External id": 2483772,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 571
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758158.415, "dur": 0.826,
    "args": {
      "External id": 2483773,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 572
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758161.497, "dur": 2.473,
    "args": {
      "External id": 2483774,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 573
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758162.266, "dur": 1.019,
    "args": {
      "External id": 2483775,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 574
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758162.852, "dur": 0.353,
    "args": {
      "External id": 2483776,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 575
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758164.413, "dur": 34.118,
    "args": {
      "External id": 2483777,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 576
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758213.579, "dur": 28.441,
    "args": {
      "External id": 2483778,"Record function id": 0, "Sequence number": 24740584, "Fwd thread id": 1, "Ev Idx": 577
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758214.532, "dur": 5.561,
    "args": {
      "External id": 2483779,"Sequence number": 24740584, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 578
    }
  },
  {
    "ph": "f", "id": 64, "pid": 1336754, "tid": 1381179, "ts": 1514656758214.532,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758218.370, "dur": 1.570,
    "args": {
      "External id": 2483780,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 579
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758218.926, "dur": 0.881,
    "args": {
      "External id": 2483781,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 580
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758222.512, "dur": 17.322,
    "args": {
      "External id": 2483782,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 581
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758246.010, "dur": 11.906,
    "args": {
      "External id": 2483783,"Record function id": 0, "Sequence number": 24740583, "Fwd thread id": 1, "Ev Idx": 582
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758247.064, "dur": 8.291,
    "args": {
      "External id": 2483784,"Sequence number": 24740583, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 583
    }
  },
  {
    "ph": "f", "id": 65, "pid": 1336754, "tid": 1381179, "ts": 1514656758247.064,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758248.010, "dur": 7.123,
    "args": {
      "External id": 2483785,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 584
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758251.011, "dur": 3.546,
    "args": {
      "External id": 2483786,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 585
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758253.935, "dur": 0.496,
    "args": {
      "External id": 2483787,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 586
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758261.864, "dur": 7.033,
    "args": {
      "External id": 2483788,"Record function id": 0, "Ev Idx": 587
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758263.199, "dur": 5.237,
    "args": {
      "External id": 2483789,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 588
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758264.190, "dur": 3.946,
    "args": {
      "External id": 2483790,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 589
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758267.029, "dur": 1.001,
    "args": {
      "External id": 2483791,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 590
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: LayerNormFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758273.111, "dur": 389.598,
    "args": {
      "External id": 2483792,"Record function id": 0, "Sequence number": 24740582, "Fwd thread id": 1, "Ev Idx": 591
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758274.258, "dur": 351.873,
    "args": {
      "External id": 2483793,"Sequence number": 24740582, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 592
    }
  },
  {
    "ph": "f", "id": 66, "pid": 1336754, "tid": 1381179, "ts": 1514656758274.258,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758309.080, "dur": 1.714,
    "args": {
      "External id": 2483794,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 593
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758309.548, "dur": 1.098,
    "args": {
      "External id": 2483795,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 594
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758326.726, "dur": 5.951,
    "args": {
      "External id": 2483796,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 595
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758342.040, "dur": 2.318,
    "args": {
      "External id": 2483797,"Record function id": 0, "Concrete Inputs": ["[132, 2048]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 596
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758500.756, "dur": 1.766,
    "args": {
      "External id": 2483798,"Record function id": 0, "Concrete Inputs": ["", "[1, -1, 2048]"], "Input type": ["float", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[132, 2048], []], "Ev Idx": 597
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758506.461, "dur": 53.333,
    "args": {
      "External id": 2483799,"Record function id": 0, "Concrete Inputs": ["", "[1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[270336, 2048, 1], [], [], []], "Input Dims": [[1, 132, 2048], [], [], []], "Ev Idx": 598
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758517.823, "dur": 0.820,
    "args": {
      "External id": 2483800,"Record function id": 0, "Concrete Inputs": ["", "[1, 1, 2048]", "[2048, 0, 1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[1, 2048], [], [], []], "Ev Idx": 599
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758566.810, "dur": 31.929,
    "args": {
      "External id": 2483801,"Record function id": 0, "Concrete Inputs": ["", "", "15", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], []], "Ev Idx": 600
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758568.435, "dur": 30.112,
    "args": {
      "External id": 2483802,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], [], []], "Ev Idx": 601
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758571.869, "dur": 5.975,
    "args": {
      "External id": 2483803,"Record function id": 0, "Concrete Inputs": ["[1, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 602
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758579.295, "dur": 18.785,
    "args": {
      "External id": 2483804,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "float", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[1, 2048], [1, 2048], []], "Ev Idx": 603
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758603.098, "dur": 2.823,
    "args": {
      "External id": 2483805,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1]], "Input Dims": [[1, 2048], [2048]], "Ev Idx": 604
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758604.233, "dur": 1.555,
    "args": {
      "External id": 2483806,"Record function id": 0, "Concrete Inputs": ["", "[2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[1, 2048], []], "Ev Idx": 605
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758614.349, "dur": 4.392,
    "args": {
      "External id": 2483807,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 606
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758617.640, "dur": 1.012,
    "args": {
      "External id": 2483808,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 607
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758638.264, "dur": 17.685,
    "args": {
      "External id": 2483809,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 608
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758673.941, "dur": 8.001,
    "args": {
      "External id": 2483810,"Record function id": 0, "Ev Idx": 609
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758675.861, "dur": 5.447,
    "args": {
      "External id": 2483811,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 610
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758677.741, "dur": 2.685,
    "args": {
      "External id": 2483812,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 611
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758678.710, "dur": 1.614,
    "args": {
      "External id": 2483813,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 612
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: AddBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758688.039, "dur": 7.251,
    "args": {
      "External id": 2483814,"Record function id": 0, "Sequence number": 24740581, "Fwd thread id": 1, "Ev Idx": 613
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "AddBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758689.393, "dur": 1.049,
    "args": {
      "External id": 2483815,"Sequence number": 24740581, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 614
    }
  },
  {
    "ph": "f", "id": 67, "pid": 1336754, "tid": 1381179, "ts": 1514656758689.393,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SwiGLULinearFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758699.586, "dur": 466.638,
    "args": {
      "External id": 2483816,"Record function id": 0, "Sequence number": 24740580, "Fwd thread id": 1, "Ev Idx": 615
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SwiGLULinearFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758700.779, "dur": 450.957,
    "args": {
      "External id": 2483817,"Sequence number": 24740580, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 616
    }
  },
  {
    "ph": "f", "id": 68, "pid": 1336754, "tid": 1381179, "ts": 1514656758700.779,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758734.627, "dur": 7.715,
    "args": {
      "External id": 2483818,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 617
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_reshape_alias", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758739.102, "dur": 3.002,
    "args": {
      "External id": 2483819,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]", "[8192, 1]"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList"], "Input Strides": [[33554432, 8192, 1], [], []], "Input Dims": [[16, 4096, 2048], [], []], "Ev Idx": 618
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758745.771, "dur": 6.680,
    "args": {
      "External id": 2483820,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 619
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758747.226, "dur": 4.467,
    "args": {
      "External id": 2483821,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 620
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758750.706, "dur": 0.811,
    "args": {
      "External id": 2483822,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 621
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758757.759, "dur": 84.168,
    "args": {
      "External id": 2483823,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8192, 1], [1, 5632], []], "Input Dims": [[65536, 2048], [5632, 2048], []], "Ev Idx": 622
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758758.739, "dur": 5.008,
    "args": {
      "External id": 2483824,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 5632]], "Input Dims": [[5632, 2048]], "Ev Idx": 623
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758759.168, "dur": 4.164,
    "args": {
      "External id": 2483825,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 5632], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 624
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758760.086, "dur": 3.151,
    "args": {
      "External id": 2483826,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[5632, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 5632], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 625
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758764.683, "dur": 76.882,
    "args": {
      "External id": 2483827,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8192, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 626
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758766.184, "dur": 74.480,
    "args": {
      "External id": 2483828,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8192, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 627
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758845.312, "dur": 4.504,
    "args": {
      "External id": 2483829,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [23068672, 5632, 1]], "Input Dims": [[65536, 5632], [16, 4096, 5632]], "Ev Idx": 628
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758848.481, "dur": 1.188,
    "args": {
      "External id": 2483830,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 629
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758884.119, "dur": 5.568,
    "args": {
      "External id": 2483831,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 630
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758890.568, "dur": 2.058,
    "args": {
      "External id": 2483832,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 631
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758893.286, "dur": 1.702,
    "args": {
      "External id": 2483833,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 632
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758927.675, "dur": 1.874,
    "args": {
      "External id": 2483834,"Record function id": 0, "Concrete Inputs": ["", "[-1, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 633
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758928.212, "dur": 1.207,
    "args": {
      "External id": 2483835,"Record function id": 0, "Concrete Inputs": ["", "[-1, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 634
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::einsum", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758953.159, "dur": 176.006,
    "args": {
      "External id": 2483836,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["", "TensorList", ""], "Input Strides": [[], [[8192, 1], [5632, 1]], []], "Input Dims": [[], [[65536, 2048], [65536, 5632]], []], "Ev Idx": 635
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758958.075, "dur": 4.289,
    "args": {
      "External id": 2483837,"Record function id": 0, "Concrete Inputs": ["", "2"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 636
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758960.641, "dur": 0.906,
    "args": {
      "External id": 2483838,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048, 1]", "[8192, 1, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[8192, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 637
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758963.727, "dur": 9.529,
    "args": {
      "External id": 2483839,"Record function id": 0, "Concrete Inputs": ["", "[1, 2, 0]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8192, 1, 1], []], "Input Dims": [[65536, 2048, 1], []], "Ev Idx": 638
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758970.112, "dur": 2.363,
    "args": {
      "External id": 2483840,"Record function id": 0, "Concrete Inputs": ["", "[2048, 1, 65536]", "[1, 1, 8192]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[65536, 2048, 1], [], [], []], "Ev Idx": 639
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758974.534, "dur": 3.388,
    "args": {
      "External id": 2483841,"Record function id": 0, "Concrete Inputs": ["", "2"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 640
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758977.190, "dur": 0.364,
    "args": {
      "External id": 2483842,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632, 1]", "[5632, 1, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[65536, 5632], [], [], []], "Ev Idx": 641
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758978.425, "dur": 4.393,
    "args": {
      "External id": 2483843,"Record function id": 0, "Concrete Inputs": ["", "[2, 1, 0]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 1], []], "Input Dims": [[65536, 5632, 1], []], "Ev Idx": 642
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656758981.781, "dur": 0.487,
    "args": {
      "External id": 2483844,"Record function id": 0, "Concrete Inputs": ["", "[1, 5632, 65536]", "[1, 1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1, 1], [], [], []], "Input Dims": [[65536, 5632, 1], [], [], []], "Ev Idx": 643
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759028.228, "dur": 2.750,
    "args": {
      "External id": 2483845,"Record function id": 0, "Concrete Inputs": ["", "[0, 2, 1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1, 1, 8192], []], "Input Dims": [[2048, 1, 65536], []], "Ev Idx": 644
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759029.526, "dur": 0.725,
    "args": {
      "External id": 2483846,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536, 1]", "[1, 8192, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1, 8192], [], [], []], "Input Dims": [[2048, 1, 65536], [], [], []], "Ev Idx": 645
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759035.926, "dur": 5.269,
    "args": {
      "External id": 2483847,"Record function id": 0, "Concrete Inputs": ["", "[1, 2048, 65536]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1, 8192, 1], []], "Input Dims": [[2048, 65536, 1], []], "Ev Idx": 646
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_reshape_alias", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759039.078, "dur": 1.953,
    "args": {
      "External id": 2483848,"Record function id": 0, "Concrete Inputs": ["", "[1, 2048, 65536]", "[2048, 1, 8192]"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList"], "Input Strides": [[1, 8192, 1], [], []], "Input Dims": [[2048, 65536, 1], [], []], "Ev Idx": 647
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759044.178, "dur": 1.731,
    "args": {
      "External id": 2483849,"Record function id": 0, "Concrete Inputs": ["", "[2, 1, 0]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1, 1, 5632], []], "Input Dims": [[1, 5632, 65536], []], "Ev Idx": 648
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759045.179, "dur": 0.486,
    "args": {
      "External id": 2483850,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632, 1]", "[5632, 1, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1, 5632], [], [], []], "Input Dims": [[1, 5632, 65536], [], [], []], "Ev Idx": 649
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759046.463, "dur": 6.123,
    "args": {
      "External id": 2483851,"Record function id": 0, "Concrete Inputs": ["", "[1, 65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 1], []], "Input Dims": [[65536, 5632, 1], []], "Ev Idx": 650
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759049.232, "dur": 3.253,
    "args": {
      "External id": 2483852,"Record function id": 0, "Concrete Inputs": ["", "[1, 65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 1], []], "Input Dims": [[65536, 5632, 1], []], "Ev Idx": 651
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759053.911, "dur": 60.337,
    "args": {
      "External id": 2483853,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1, 8192], [369098752, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632]], "Ev Idx": 652
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759116.132, "dur": 4.003,
    "args": {
      "External id": 2483854,"Record function id": 0, "Concrete Inputs": ["", "[2048, 1, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[11534336, 5632, 1], []], "Input Dims": [[1, 2048, 5632], []], "Ev Idx": 653
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759120.670, "dur": 4.372,
    "args": {
      "External id": 2483855,"Record function id": 0, "Concrete Inputs": ["", "[0, 2, 1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 5632, 1], []], "Input Dims": [[2048, 1, 5632], []], "Ev Idx": 654
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759124.090, "dur": 0.411,
    "args": {
      "External id": 2483856,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632, 1]", "[5632, 1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 5632, 1], [], [], []], "Input Dims": [[2048, 1, 5632], [], [], []], "Ev Idx": 655
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759127.045, "dur": 1.124,
    "args": {
      "External id": 2483857,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 5632], []], "Input Dims": [[2048, 5632, 1], []], "Ev Idx": 656
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759178.194, "dur": 8.538,
    "args": {
      "External id": 2483858,"Record function id": 0, "Ev Idx": 657
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759180.381, "dur": 5.630,
    "args": {
      "External id": 2483859,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 658
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759182.232, "dur": 2.959,
    "args": {
      "External id": 2483860,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 659
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759183.223, "dur": 1.858,
    "args": {
      "External id": 2483861,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 660
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759190.805, "dur": 7.397,
    "args": {
      "External id": 2483862,"Record function id": 0, "Sequence number": 24740579, "Fwd thread id": 1, "Ev Idx": 661
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759191.784, "dur": 3.650,
    "args": {
      "External id": 2483863,"Sequence number": 24740579, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[23068672, 5632, 1]], "Input Dims": [[16, 4096, 5632]], "Ev Idx": 662
    }
  },
  {
    "ph": "f", "id": 69, "pid": 1336754, "tid": 1381179, "ts": 1514656759191.784,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759193.727, "dur": 1.491,
    "args": {
      "External id": 2483864,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 663
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759194.292, "dur": 0.785,
    "args": {
      "External id": 2483865,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 664
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759201.529, "dur": 130.718,
    "args": {
      "External id": 2483866,"Record function id": 0, "Sequence number": 24740578, "Fwd thread id": 1, "Ev Idx": 665
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759202.382, "dur": 121.723,
    "args": {
      "External id": 2483867,"Sequence number": 24740578, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 666
    }
  },
  {
    "ph": "f", "id": 70, "pid": 1336754, "tid": 1381179, "ts": 1514656759202.382,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759209.687, "dur": 6.376,
    "args": {
      "External id": 2483868,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 667
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759211.004, "dur": 4.578,
    "args": {
      "External id": 2483869,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[65536, 5632], [], []], "Ev Idx": 668
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759214.770, "dur": 0.647,
    "args": {
      "External id": 2483870,"Record function id": 0, "Concrete Inputs": ["", "[5632, 65536]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[65536, 5632], [], [], []], "Ev Idx": 669
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759217.315, "dur": 44.129,
    "args": {
      "External id": 2483871,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048]], "Ev Idx": 670
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759262.474, "dur": 6.124,
    "args": {
      "External id": 2483872,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 671
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759263.082, "dur": 4.722,
    "args": {
      "External id": 2483873,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 672
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759266.554, "dur": 1.100,
    "args": {
      "External id": 2483874,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 673
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759270.554, "dur": 2.541,
    "args": {
      "External id": 2483875,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 674
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759271.372, "dur": 1.289,
    "args": {
      "External id": 2483876,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 675
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759272.001, "dur": 0.597,
    "args": {
      "External id": 2483877,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 676
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759273.567, "dur": 49.671,
    "args": {
      "External id": 2483878,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048]], "Ev Idx": 677
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759336.738, "dur": 7.834,
    "args": {
      "External id": 2483879,"Record function id": 0, "Sequence number": 24740577, "Fwd thread id": 1, "Ev Idx": 678
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759337.711, "dur": 5.259,
    "args": {
      "External id": 2483880,"Sequence number": 24740577, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 679
    }
  },
  {
    "ph": "f", "id": 71, "pid": 1336754, "tid": 1381179, "ts": 1514656759337.711,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759341.120, "dur": 1.683,
    "args": {
      "External id": 2483881,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 680
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759341.640, "dur": 1.024,
    "args": {
      "External id": 2483882,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 681
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759347.886, "dur": 9.845,
    "args": {
      "External id": 2483883,"Record function id": 0, "Sequence number": 24740576, "Fwd thread id": 1, "Ev Idx": 682
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759350.817, "dur": 3.679,
    "args": {
      "External id": 2483884,"Sequence number": 24740576, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 683
    }
  },
  {
    "ph": "f", "id": 72, "pid": 1336754, "tid": 1381179, "ts": 1514656759350.817,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759352.037, "dur": 2.222,
    "args": {
      "External id": 2483885,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 684
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759352.562, "dur": 1.194,
    "args": {
      "External id": 2483886,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 685
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759353.297, "dur": 0.349,
    "args": {
      "External id": 2483887,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 686
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759361.356, "dur": 5.056,
    "args": {
      "External id": 2483888,"Record function id": 0, "Ev Idx": 687
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759362.673, "dur": 3.238,
    "args": {
      "External id": 2483889,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 688
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759363.957, "dur": 1.617,
    "args": {
      "External id": 2483890,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 689
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759364.334, "dur": 1.152,
    "args": {
      "External id": 2483891,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 690
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759369.456, "dur": 10.230,
    "args": {
      "External id": 2483892,"Record function id": 0, "Sequence number": 24740575, "Fwd thread id": 1, "Ev Idx": 691
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759372.795, "dur": 4.181,
    "args": {
      "External id": 2483893,"Sequence number": 24740575, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[23068672, 5632, 1]], "Input Dims": [[16, 4096, 5632]], "Ev Idx": 692
    }
  },
  {
    "ph": "f", "id": 73, "pid": 1336754, "tid": 1381179, "ts": 1514656759372.795,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759373.950, "dur": 2.875,
    "args": {
      "External id": 2483894,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 693
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759376.061, "dur": 0.642,
    "args": {
      "External id": 2483895,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 694
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759382.639, "dur": 123.530,
    "args": {
      "External id": 2483896,"Record function id": 0, "Sequence number": 24740574, "Fwd thread id": 1, "Ev Idx": 695
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759383.470, "dur": 112.443,
    "args": {
      "External id": 2483897,"Sequence number": 24740574, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 696
    }
  },
  {
    "ph": "f", "id": 74, "pid": 1336754, "tid": 1381179, "ts": 1514656759383.470,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759387.094, "dur": 5.999,
    "args": {
      "External id": 2483898,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 697
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759388.009, "dur": 4.635,
    "args": {
      "External id": 2483899,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[65536, 5632], [], []], "Ev Idx": 698
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759391.931, "dur": 0.589,
    "args": {
      "External id": 2483900,"Record function id": 0, "Concrete Inputs": ["", "[5632, 65536]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[65536, 5632], [], [], []], "Ev Idx": 699
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759396.254, "dur": 37.603,
    "args": {
      "External id": 2483901,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048]], "Ev Idx": 700
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759434.898, "dur": 5.348,
    "args": {
      "External id": 2483902,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 701
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759435.626, "dur": 4.106,
    "args": {
      "External id": 2483903,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 702
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759437.360, "dur": 2.219,
    "args": {
      "External id": 2483904,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 703
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759441.552, "dur": 9.233,
    "args": {
      "External id": 2483905,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 704
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759442.252, "dur": 8.010,
    "args": {
      "External id": 2483906,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 705
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759447.829, "dur": 2.361,
    "args": {
      "External id": 2483907,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 706
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759451.375, "dur": 43.868,
    "args": {
      "External id": 2483908,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048]], "Ev Idx": 707
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759510.605, "dur": 49.678,
    "args": {
      "External id": 2483909,"Record function id": 0, "Sequence number": 24740573, "Fwd thread id": 1, "Ev Idx": 708
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759511.473, "dur": 3.199,
    "args": {
      "External id": 2483910,"Sequence number": 24740573, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 709
    }
  },
  {
    "ph": "f", "id": 75, "pid": 1336754, "tid": 1381179, "ts": 1514656759511.473,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759513.068, "dur": 1.445,
    "args": {
      "External id": 2483911,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 710
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759513.525, "dur": 0.854,
    "args": {
      "External id": 2483912,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 711
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759516.887, "dur": 39.446,
    "args": {
      "External id": 2483913,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 712
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759566.240, "dur": 12.666,
    "args": {
      "External id": 2483914,"Record function id": 0, "Sequence number": 24740572, "Fwd thread id": 1, "Ev Idx": 713
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759567.706, "dur": 8.901,
    "args": {
      "External id": 2483915,"Sequence number": 24740572, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 714
    }
  },
  {
    "ph": "f", "id": 76, "pid": 1336754, "tid": 1381179, "ts": 1514656759567.706,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759568.614, "dur": 7.765,
    "args": {
      "External id": 2483916,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 715
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759569.671, "dur": 5.983,
    "args": {
      "External id": 2483917,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 716
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759574.912, "dur": 0.620,
    "args": {
      "External id": 2483918,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 717
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759582.979, "dur": 5.399,
    "args": {
      "External id": 2483919,"Record function id": 0, "Ev Idx": 718
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759584.130, "dur": 3.762,
    "args": {
      "External id": 2483920,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 719
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759585.442, "dur": 1.967,
    "args": {
      "External id": 2483921,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 720
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759585.999, "dur": 1.339,
    "args": {
      "External id": 2483922,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 721
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: LayerNormFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759594.111, "dur": 462.450,
    "args": {
      "External id": 2483923,"Record function id": 0, "Sequence number": 24740571, "Fwd thread id": 1, "Ev Idx": 722
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759595.262, "dur": 380.288,
    "args": {
      "External id": 2483924,"Sequence number": 24740571, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [33554432, 8192, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 723
    }
  },
  {
    "ph": "f", "id": 77, "pid": 1336754, "tid": 1381179, "ts": 1514656759595.262,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::contiguous", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759618.623, "dur": 38.345,
    "args": {
      "External id": 2483925,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 724
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759619.945, "dur": 36.835,
    "args": {
      "External id": 2483926,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 725
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759622.465, "dur": 9.727,
    "args": {
      "External id": 2483927,"Record function id": 0, "Concrete Inputs": ["", "15", "0", "", "", "0"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[33554432, 8192, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], [], []], "Ev Idx": 726
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759628.621, "dur": 3.140,
    "args": {
      "External id": 2483928,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 2048]", "15", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 727
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759633.495, "dur": 22.733,
    "args": {
      "External id": 2483929,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 728
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759668.019, "dur": 4.492,
    "args": {
      "External id": 2483930,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 729
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759671.121, "dur": 1.171,
    "args": {
      "External id": 2483931,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 730
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759675.947, "dur": 4.241,
    "args": {
      "External id": 2483932,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 731
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759676.662, "dur": 3.409,
    "args": {
      "External id": 2483933,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 732
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759691.864, "dur": 1.929,
    "args": {
      "External id": 2483934,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 733
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759704.326, "dur": 1.779,
    "args": {
      "External id": 2483935,"Record function id": 0, "Concrete Inputs": ["[132, 2048]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 734
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759871.316, "dur": 2.222,
    "args": {
      "External id": 2483936,"Record function id": 0, "Concrete Inputs": ["", "[1, -1, 2048]"], "Input type": ["float", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[132, 2048], []], "Ev Idx": 735
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759877.584, "dur": 31.637,
    "args": {
      "External id": 2483937,"Record function id": 0, "Concrete Inputs": ["", "[1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[270336, 2048, 1], [], [], []], "Input Dims": [[1, 132, 2048], [], [], []], "Ev Idx": 736
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759885.714, "dur": 0.791,
    "args": {
      "External id": 2483938,"Record function id": 0, "Concrete Inputs": ["", "[1, 1, 2048]", "[2048, 0, 1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[1, 2048], [], [], []], "Ev Idx": 737
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759914.338, "dur": 28.638,
    "args": {
      "External id": 2483939,"Record function id": 0, "Concrete Inputs": ["", "", "15", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], []], "Ev Idx": 738
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759915.827, "dur": 26.955,
    "args": {
      "External id": 2483940,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], [], []], "Ev Idx": 739
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759921.954, "dur": 4.124,
    "args": {
      "External id": 2483941,"Record function id": 0, "Concrete Inputs": ["[1, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 740
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759927.136, "dur": 15.193,
    "args": {
      "External id": 2483942,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "float", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[1, 2048], [1, 2048], []], "Ev Idx": 741
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759947.179, "dur": 3.918,
    "args": {
      "External id": 2483943,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1]], "Input Dims": [[1, 2048], [2048]], "Ev Idx": 742
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759949.997, "dur": 0.986,
    "args": {
      "External id": 2483944,"Record function id": 0, "Concrete Inputs": ["", "[2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[1, 2048], []], "Ev Idx": 743
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759956.791, "dur": 2.005,
    "args": {
      "External id": 2483945,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 744
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759957.331, "dur": 1.358,
    "args": {
      "External id": 2483946,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 745
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759962.258, "dur": 4.256,
    "args": {
      "External id": 2483947,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 746
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656759962.712, "dur": 3.707,
    "args": {
      "External id": 2483948,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 747
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760031.689, "dur": 22.240,
    "args": {
      "External id": 2483949,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 748
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760069.026, "dur": 10.406,
    "args": {
      "External id": 2483950,"Record function id": 0, "Ev Idx": 749
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760071.008, "dur": 7.728,
    "args": {
      "External id": 2483951,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 750
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760072.928, "dur": 4.878,
    "args": {
      "External id": 2483952,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 751
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760075.731, "dur": 1.984,
    "args": {
      "External id": 2483953,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 752
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760083.129, "dur": 8.315,
    "args": {
      "External id": 2483954,"Record function id": 0, "Sequence number": 24740570, "Fwd thread id": 1, "Ev Idx": 753
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760084.051, "dur": 4.198,
    "args": {
      "External id": 2483955,"Sequence number": 24740570, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 754
    }
  },
  {
    "ph": "f", "id": 78, "pid": 1336754, "tid": 1381179, "ts": 1514656760084.051,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760085.697, "dur": 2.336,
    "args": {
      "External id": 2483956,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 755
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760086.401, "dur": 1.508,
    "args": {
      "External id": 2483957,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 756
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760094.913, "dur": 137.436,
    "args": {
      "External id": 2483958,"Record function id": 0, "Sequence number": 24740569, "Fwd thread id": 1, "Ev Idx": 757
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760096.016, "dur": 129.920,
    "args": {
      "External id": 2483959,"Sequence number": 24740569, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 758
    }
  },
  {
    "ph": "f", "id": 79, "pid": 1336754, "tid": 1381179, "ts": 1514656760096.016,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760099.247, "dur": 8.726,
    "args": {
      "External id": 2483960,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 759
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760102.874, "dur": 4.452,
    "args": {
      "External id": 2483961,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 760
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760106.320, "dur": 0.803,
    "args": {
      "External id": 2483962,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 761
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760109.271, "dur": 62.771,
    "args": {
      "External id": 2483963,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 762
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760173.083, "dur": 5.074,
    "args": {
      "External id": 2483964,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 763
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760173.696, "dur": 3.845,
    "args": {
      "External id": 2483965,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 764
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760174.630, "dur": 2.764,
    "args": {
      "External id": 2483966,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 765
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760181.596, "dur": 2.490,
    "args": {
      "External id": 2483967,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 766
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760182.624, "dur": 0.955,
    "args": {
      "External id": 2483968,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 767
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760183.143, "dur": 0.356,
    "args": {
      "External id": 2483969,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 768
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760184.550, "dur": 40.517,
    "args": {
      "External id": 2483970,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 769
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760236.905, "dur": 11.558,
    "args": {
      "External id": 2483971,"Record function id": 0, "Sequence number": 24740568, "Fwd thread id": 1, "Ev Idx": 770
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760237.826, "dur": 8.134,
    "args": {
      "External id": 2483972,"Sequence number": 24740568, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 771
    }
  },
  {
    "ph": "f", "id": 80, "pid": 1336754, "tid": 1381179, "ts": 1514656760237.826,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760241.199, "dur": 4.599,
    "args": {
      "External id": 2483973,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 772
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760241.933, "dur": 3.720,
    "args": {
      "External id": 2483974,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 773
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760253.834, "dur": 9.324,
    "args": {
      "External id": 2483975,"Record function id": 0, "Sequence number": 24740567, "Fwd thread id": 1, "Ev Idx": 774
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760256.608, "dur": 3.875,
    "args": {
      "External id": 2483976,"Sequence number": 24740567, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 775
    }
  },
  {
    "ph": "f", "id": 81, "pid": 1336754, "tid": 1381179, "ts": 1514656760256.608,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760257.491, "dur": 2.774,
    "args": {
      "External id": 2483977,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 776
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760258.323, "dur": 1.445,
    "args": {
      "External id": 2483978,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 777
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760259.205, "dur": 0.427,
    "args": {
      "External id": 2483979,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 778
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760266.905, "dur": 4.877,
    "args": {
      "External id": 2483980,"Record function id": 0, "Ev Idx": 779
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760268.240, "dur": 3.078,
    "args": {
      "External id": 2483981,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 780
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760269.141, "dur": 1.964,
    "args": {
      "External id": 2483982,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 781
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760270.056, "dur": 0.931,
    "args": {
      "External id": 2483983,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 782
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760274.834, "dur": 10.744,
    "args": {
      "External id": 2483984,"Record function id": 0, "Sequence number": 24740566, "Fwd thread id": 1, "Ev Idx": 783
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760275.685, "dur": 7.353,
    "args": {
      "External id": 2483985,"Sequence number": 24740566, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 784
    }
  },
  {
    "ph": "f", "id": 82, "pid": 1336754, "tid": 1381179, "ts": 1514656760275.685,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760279.114, "dur": 3.756,
    "args": {
      "External id": 2483986,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 785
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760281.302, "dur": 1.437,
    "args": {
      "External id": 2483987,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 786
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: FlashAttnFuncBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760289.798, "dur": 313.885,
    "args": {
      "External id": 2483988,"Record function id": 0, "Sequence number": 24740565, "Fwd thread id": 1, "Ev Idx": 787
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FlashAttnFuncBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760290.904, "dur": 289.223,
    "args": {
      "External id": 2483989,"Sequence number": 24740565, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 788
    }
  },
  {
    "ph": "f", "id": 83, "pid": 1336754, "tid": 1381179, "ts": 1514656760290.904,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760307.660, "dur": 5.976,
    "args": {
      "External id": 2483990,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 789
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760309.799, "dur": 3.411,
    "args": {
      "External id": 2483991,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 790
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760315.523, "dur": 5.200,
    "args": {
      "External id": 2483992,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 791
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760318.498, "dur": 2.038,
    "args": {
      "External id": 2483993,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 792
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760322.656, "dur": 4.364,
    "args": {
      "External id": 2483994,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 793
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760323.449, "dur": 3.395,
    "args": {
      "External id": 2483995,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 794
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760353.784, "dur": 198.846,
    "args": {
      "External id": 2483996,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 795
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760440.609, "dur": 3.204,
    "args": {
      "External id": 2483997,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 796
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760445.516, "dur": 3.985,
    "args": {
      "External id": 2483998,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 797
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760566.883, "dur": 3.907,
    "args": {
      "External id": 2483999,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 798
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760573.719, "dur": 1.002,
    "args": {
      "External id": 2484000,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 799
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760576.683, "dur": 0.700,
    "args": {
      "External id": 2484001,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 800
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RotaryEmbeddingFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760612.201, "dur": 235.235,
    "args": {
      "External id": 2484002,"Record function id": 0, "Sequence number": 24740564, "Fwd thread id": 1, "Ev Idx": 801
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760613.628, "dur": 225.892,
    "args": {
      "External id": 2484003,"Sequence number": 24740564, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 802
    }
  },
  {
    "ph": "f", "id": 84, "pid": 1336754, "tid": 1381179, "ts": 1514656760613.628,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760637.742, "dur": 49.520,
    "args": {
      "External id": 2484004,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 803
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760640.581, "dur": 3.621,
    "args": {
      "External id": 2484005,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 804
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760645.792, "dur": 40.675,
    "args": {
      "External id": 2484006,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], []], "Ev Idx": 805
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760696.978, "dur": 6.129,
    "args": {
      "External id": 2484007,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 806
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760700.613, "dur": 2.127,
    "args": {
      "External id": 2484008,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 807
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RotaryEmbeddingFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760854.549, "dur": 194.869,
    "args": {
      "External id": 2484009,"Record function id": 0, "Sequence number": 24740563, "Fwd thread id": 1, "Ev Idx": 808
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760855.976, "dur": 184.602,
    "args": {
      "External id": 2484010,"Sequence number": 24740563, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 809
    }
  },
  {
    "ph": "f", "id": 85, "pid": 1336754, "tid": 1381179, "ts": 1514656760855.976,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760868.010, "dur": 30.342,
    "args": {
      "External id": 2484011,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 810
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760870.061, "dur": 2.094,
    "args": {
      "External id": 2484012,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 811
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760873.410, "dur": 24.316,
    "args": {
      "External id": 2484013,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], []], "Ev Idx": 812
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760905.300, "dur": 8.355,
    "args": {
      "External id": 2484014,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 813
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656760911.392, "dur": 1.944,
    "args": {
      "External id": 2484015,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 814
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761058.349, "dur": 15.870,
    "args": {
      "External id": 2484016,"Record function id": 0, "Sequence number": 24740562, "Fwd thread id": 1, "Ev Idx": 815
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761059.953, "dur": 11.009,
    "args": {
      "External id": 2484017,"Sequence number": 24740562, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 816
    }
  },
  {
    "ph": "f", "id": 86, "pid": 1336754, "tid": 1381179, "ts": 1514656761059.953,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761062.665, "dur": 8.038,
    "args": {
      "External id": 2484018,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 817
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761063.775, "dur": 6.758,
    "args": {
      "External id": 2484019,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 818
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761077.744, "dur": 11.554,
    "args": {
      "External id": 2484020,"Record function id": 0, "Sequence number": 24740561, "Fwd thread id": 1, "Ev Idx": 819
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761081.646, "dur": 5.020,
    "args": {
      "External id": 2484021,"Sequence number": 24740561, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 820
    }
  },
  {
    "ph": "f", "id": 87, "pid": 1336754, "tid": 1381179, "ts": 1514656761081.646,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761082.691, "dur": 3.834,
    "args": {
      "External id": 2484022,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 821
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761085.354, "dur": 1.077,
    "args": {
      "External id": 2484023,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 822
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761094.704, "dur": 7.366,
    "args": {
      "External id": 2484024,"Record function id": 0, "Sequence number": 24740560, "Fwd thread id": 1, "Ev Idx": 823
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761095.365, "dur": 3.454,
    "args": {
      "External id": 2484025,"Sequence number": 24740560, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 824
    }
  },
  {
    "ph": "f", "id": 88, "pid": 1336754, "tid": 1381179, "ts": 1514656761095.365,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761097.229, "dur": 1.431,
    "args": {
      "External id": 2484026,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 825
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761097.698, "dur": 0.880,
    "args": {
      "External id": 2484027,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 826
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761105.443, "dur": 11.548,
    "args": {
      "External id": 2484028,"Record function id": 0, "Sequence number": 24740559, "Fwd thread id": 1, "Ev Idx": 827
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761106.364, "dur": 7.666,
    "args": {
      "External id": 2484029,"Sequence number": 24740559, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 828
    }
  },
  {
    "ph": "f", "id": 89, "pid": 1336754, "tid": 1381179, "ts": 1514656761106.364,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761107.964, "dur": 5.916,
    "args": {
      "External id": 2484030,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 829
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761112.958, "dur": 0.827,
    "args": {
      "External id": 2484031,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 830
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761120.335, "dur": 148.285,
    "args": {
      "External id": 2484032,"Record function id": 0, "Sequence number": 24740558, "Fwd thread id": 1, "Ev Idx": 831
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761121.677, "dur": 139.091,
    "args": {
      "External id": 2484033,"Sequence number": 24740558, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 832
    }
  },
  {
    "ph": "f", "id": 90, "pid": 1336754, "tid": 1381179, "ts": 1514656761121.677,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761126.479, "dur": 5.994,
    "args": {
      "External id": 2484034,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 833
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761128.219, "dur": 3.641,
    "args": {
      "External id": 2484035,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 834
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761130.254, "dur": 1.262,
    "args": {
      "External id": 2484036,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 835
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761134.009, "dur": 66.455,
    "args": {
      "External id": 2484037,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 836
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761201.653, "dur": 6.473,
    "args": {
      "External id": 2484038,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 837
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761204.527, "dur": 2.893,
    "args": {
      "External id": 2484039,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 838
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761205.904, "dur": 1.346,
    "args": {
      "External id": 2484040,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 839
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761209.727, "dur": 5.578,
    "args": {
      "External id": 2484041,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 840
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761211.162, "dur": 3.407,
    "args": {
      "External id": 2484042,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 841
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761213.731, "dur": 0.773,
    "args": {
      "External id": 2484043,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 842
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761215.815, "dur": 44.187,
    "args": {
      "External id": 2484044,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 843
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761273.270, "dur": 8.590,
    "args": {
      "External id": 2484045,"Record function id": 0, "Sequence number": 24740557, "Fwd thread id": 1, "Ev Idx": 844
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761274.088, "dur": 5.636,
    "args": {
      "External id": 2484046,"Sequence number": 24740557, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 845
    }
  },
  {
    "ph": "f", "id": 91, "pid": 1336754, "tid": 1381179, "ts": 1514656761274.088,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761277.703, "dur": 1.865,
    "args": {
      "External id": 2484047,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 846
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761278.547, "dur": 0.916,
    "args": {
      "External id": 2484048,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 847
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761285.158, "dur": 6.497,
    "args": {
      "External id": 2484049,"Record function id": 0, "Sequence number": 24740556, "Fwd thread id": 1, "Ev Idx": 848
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761285.961, "dur": 3.690,
    "args": {
      "External id": 2484050,"Sequence number": 24740556, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 849
    }
  },
  {
    "ph": "f", "id": 92, "pid": 1336754, "tid": 1381179, "ts": 1514656761285.961,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761287.015, "dur": 2.443,
    "args": {
      "External id": 2484051,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 850
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761287.522, "dur": 1.470,
    "args": {
      "External id": 2484052,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 851
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761288.467, "dur": 0.411,
    "args": {
      "External id": 2484053,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 852
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761299.560, "dur": 10.801,
    "args": {
      "External id": 2484054,"Record function id": 0, "Ev Idx": 853
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761301.141, "dur": 8.479,
    "args": {
      "External id": 2484055,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 854
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761303.534, "dur": 5.580,
    "args": {
      "External id": 2484056,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 855
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761306.520, "dur": 2.451,
    "args": {
      "External id": 2484057,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 856
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761313.667, "dur": 7.335,
    "args": {
      "External id": 2484058,"Record function id": 0, "Sequence number": 24740555, "Fwd thread id": 1, "Ev Idx": 857
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761314.889, "dur": 3.313,
    "args": {
      "External id": 2484059,"Sequence number": 24740555, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 858
    }
  },
  {
    "ph": "f", "id": 93, "pid": 1336754, "tid": 1381179, "ts": 1514656761314.889,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761316.598, "dur": 1.469,
    "args": {
      "External id": 2484060,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 859
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761317.267, "dur": 0.671,
    "args": {
      "External id": 2484061,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 860
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761324.018, "dur": 117.624,
    "args": {
      "External id": 2484062,"Record function id": 0, "Sequence number": 24740554, "Fwd thread id": 1, "Ev Idx": 861
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761324.973, "dur": 110.523,
    "args": {
      "External id": 2484063,"Sequence number": 24740554, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 862
    }
  },
  {
    "ph": "f", "id": 94, "pid": 1336754, "tid": 1381179, "ts": 1514656761324.973,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761328.058, "dur": 4.101,
    "args": {
      "External id": 2484064,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 863
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761330.502, "dur": 1.166,
    "args": {
      "External id": 2484065,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 864
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761331.157, "dur": 0.353,
    "args": {
      "External id": 2484066,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 865
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761332.774, "dur": 27.024,
    "args": {
      "External id": 2484067,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 866
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761360.830, "dur": 4.908,
    "args": {
      "External id": 2484068,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 867
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761361.505, "dur": 3.720,
    "args": {
      "External id": 2484069,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 868
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761364.239, "dur": 0.870,
    "args": {
      "External id": 2484070,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 869
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761369.770, "dur": 5.870,
    "args": {
      "External id": 2484071,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 870
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761371.169, "dur": 3.956,
    "args": {
      "External id": 2484072,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 871
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761371.841, "dur": 3.216,
    "args": {
      "External id": 2484073,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 872
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761376.035, "dur": 58.797,
    "args": {
      "External id": 2484074,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 873
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761446.179, "dur": 38.269,
    "args": {
      "External id": 2484075,"Record function id": 0, "Sequence number": 24740553, "Fwd thread id": 1, "Ev Idx": 874
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761446.993, "dur": 5.616,
    "args": {
      "External id": 2484076,"Sequence number": 24740553, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 875
    }
  },
  {
    "ph": "f", "id": 95, "pid": 1336754, "tid": 1381179, "ts": 1514656761446.993,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761448.404, "dur": 4.049,
    "args": {
      "External id": 2484077,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 876
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761451.419, "dur": 0.921,
    "args": {
      "External id": 2484078,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 877
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761455.752, "dur": 24.446,
    "args": {
      "External id": 2484079,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 878
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761487.959, "dur": 7.390,
    "args": {
      "External id": 2484080,"Record function id": 0, "Sequence number": 24740552, "Fwd thread id": 1, "Ev Idx": 879
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761489.127, "dur": 4.363,
    "args": {
      "External id": 2484081,"Sequence number": 24740552, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 880
    }
  },
  {
    "ph": "f", "id": 96, "pid": 1336754, "tid": 1381179, "ts": 1514656761489.127,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761490.083, "dur": 3.218,
    "args": {
      "External id": 2484082,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 881
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761490.801, "dur": 1.956,
    "args": {
      "External id": 2484083,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 882
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761492.106, "dur": 0.527,
    "args": {
      "External id": 2484084,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 883
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761499.219, "dur": 7.341,
    "args": {
      "External id": 2484085,"Record function id": 0, "Ev Idx": 884
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761500.503, "dur": 5.540,
    "args": {
      "External id": 2484086,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 885
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761501.569, "dur": 3.958,
    "args": {
      "External id": 2484087,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 886
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761504.071, "dur": 1.299,
    "args": {
      "External id": 2484088,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 887
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761509.910, "dur": 9.545,
    "args": {
      "External id": 2484089,"Record function id": 0, "Sequence number": 24740551, "Fwd thread id": 1, "Ev Idx": 888
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761510.923, "dur": 5.570,
    "args": {
      "External id": 2484090,"Sequence number": 24740551, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 889
    }
  },
  {
    "ph": "f", "id": 97, "pid": 1336754, "tid": 1381179, "ts": 1514656761510.923,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761512.011, "dur": 4.345,
    "args": {
      "External id": 2484091,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 890
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761515.484, "dur": 0.738,
    "args": {
      "External id": 2484092,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 891
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761524.448, "dur": 142.343,
    "args": {
      "External id": 2484093,"Record function id": 0, "Sequence number": 24740550, "Fwd thread id": 1, "Ev Idx": 892
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761525.285, "dur": 130.403,
    "args": {
      "External id": 2484094,"Sequence number": 24740550, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 893
    }
  },
  {
    "ph": "f", "id": 98, "pid": 1336754, "tid": 1381179, "ts": 1514656761525.285,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761543.128, "dur": 5.952,
    "args": {
      "External id": 2484095,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 894
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761543.979, "dur": 4.424,
    "args": {
      "External id": 2484096,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 895
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761547.382, "dur": 0.774,
    "args": {
      "External id": 2484097,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 896
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761549.776, "dur": 35.652,
    "args": {
      "External id": 2484098,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 897
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761586.388, "dur": 2.994,
    "args": {
      "External id": 2484099,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 898
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761586.914, "dur": 1.899,
    "args": {
      "External id": 2484100,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 899
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761587.696, "dur": 0.975,
    "args": {
      "External id": 2484101,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 900
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761590.719, "dur": 8.967,
    "args": {
      "External id": 2484102,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 901
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761595.826, "dur": 3.452,
    "args": {
      "External id": 2484103,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 902
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761598.770, "dur": 0.438,
    "args": {
      "External id": 2484104,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 903
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761600.185, "dur": 54.840,
    "args": {
      "External id": 2484105,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 904
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761672.623, "dur": 26.469,
    "args": {
      "External id": 2484106,"Record function id": 0, "Sequence number": 24740549, "Fwd thread id": 1, "Ev Idx": 905
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761673.723, "dur": 3.738,
    "args": {
      "External id": 2484107,"Sequence number": 24740549, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 906
    }
  },
  {
    "ph": "f", "id": 99, "pid": 1336754, "tid": 1381179, "ts": 1514656761673.723,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761675.256, "dur": 2.059,
    "args": {
      "External id": 2484108,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 907
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761676.074, "dur": 1.083,
    "args": {
      "External id": 2484109,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 908
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761680.003, "dur": 16.811,
    "args": {
      "External id": 2484110,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 909
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761702.800, "dur": 11.880,
    "args": {
      "External id": 2484111,"Record function id": 0, "Sequence number": 24740548, "Fwd thread id": 1, "Ev Idx": 910
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761705.602, "dur": 6.396,
    "args": {
      "External id": 2484112,"Sequence number": 24740548, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 911
    }
  },
  {
    "ph": "f", "id": 100, "pid": 1336754, "tid": 1381179, "ts": 1514656761705.602,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761706.703, "dur": 5.081,
    "args": {
      "External id": 2484113,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 912
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761707.621, "dur": 3.566,
    "args": {
      "External id": 2484114,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 913
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761710.443, "dur": 0.627,
    "args": {
      "External id": 2484115,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 914
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761718.951, "dur": 5.169,
    "args": {
      "External id": 2484116,"Record function id": 0, "Ev Idx": 915
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761720.247, "dur": 3.380,
    "args": {
      "External id": 2484117,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 916
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761721.094, "dur": 1.944,
    "args": {
      "External id": 2484118,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 917
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761721.558, "dur": 1.363,
    "args": {
      "External id": 2484119,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 918
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: LayerNormFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761728.460, "dur": 410.635,
    "args": {
      "External id": 2484120,"Record function id": 0, "Sequence number": 24740547, "Fwd thread id": 1, "Ev Idx": 919
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761729.554, "dur": 372.159,
    "args": {
      "External id": 2484121,"Sequence number": 24740547, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 920
    }
  },
  {
    "ph": "f", "id": 101, "pid": 1336754, "tid": 1381179, "ts": 1514656761729.554,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761763.166, "dur": 3.779,
    "args": {
      "External id": 2484122,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 921
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761765.702, "dur": 1.011,
    "args": {
      "External id": 2484123,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 922
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761780.475, "dur": 5.867,
    "args": {
      "External id": 2484124,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 923
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761795.446, "dur": 1.967,
    "args": {
      "External id": 2484125,"Record function id": 0, "Concrete Inputs": ["[132, 2048]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 924
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761945.898, "dur": 1.711,
    "args": {
      "External id": 2484126,"Record function id": 0, "Concrete Inputs": ["", "[1, -1, 2048]"], "Input type": ["float", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[132, 2048], []], "Ev Idx": 925
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761951.456, "dur": 76.269,
    "args": {
      "External id": 2484127,"Record function id": 0, "Concrete Inputs": ["", "[1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[270336, 2048, 1], [], [], []], "Input Dims": [[1, 132, 2048], [], [], []], "Ev Idx": 926
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656761961.562, "dur": 0.813,
    "args": {
      "External id": 2484128,"Record function id": 0, "Concrete Inputs": ["", "[1, 1, 2048]", "[2048, 0, 1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[1, 2048], [], [], []], "Ev Idx": 927
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762039.582, "dur": 39.181,
    "args": {
      "External id": 2484129,"Record function id": 0, "Concrete Inputs": ["", "", "15", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], []], "Ev Idx": 928
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762041.396, "dur": 37.168,
    "args": {
      "External id": 2484130,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], [], []], "Ev Idx": 929
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762048.978, "dur": 7.144,
    "args": {
      "External id": 2484131,"Record function id": 0, "Concrete Inputs": ["[1, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 930
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762057.837, "dur": 20.174,
    "args": {
      "External id": 2484132,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "float", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[1, 2048], [1, 2048], []], "Ev Idx": 931
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762083.138, "dur": 2.601,
    "args": {
      "External id": 2484133,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1]], "Input Dims": [[1, 2048], [2048]], "Ev Idx": 932
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762084.156, "dur": 1.472,
    "args": {
      "External id": 2484134,"Record function id": 0, "Concrete Inputs": ["", "[2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[1, 2048], []], "Ev Idx": 933
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762092.529, "dur": 1.534,
    "args": {
      "External id": 2484135,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 934
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762093.128, "dur": 0.838,
    "args": {
      "External id": 2484136,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 935
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762117.070, "dur": 17.148,
    "args": {
      "External id": 2484137,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 936
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762152.277, "dur": 11.794,
    "args": {
      "External id": 2484138,"Record function id": 0, "Ev Idx": 937
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762156.905, "dur": 6.428,
    "args": {
      "External id": 2484139,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 938
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762158.952, "dur": 3.214,
    "args": {
      "External id": 2484140,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 939
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762160.005, "dur": 2.076,
    "args": {
      "External id": 2484141,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 940
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: AddBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762167.864, "dur": 7.365,
    "args": {
      "External id": 2484142,"Record function id": 0, "Sequence number": 24740546, "Fwd thread id": 1, "Ev Idx": 941
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "AddBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762169.188, "dur": 1.243,
    "args": {
      "External id": 2484143,"Sequence number": 24740546, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 942
    }
  },
  {
    "ph": "f", "id": 102, "pid": 1336754, "tid": 1381179, "ts": 1514656762169.188,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SwiGLULinearFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762179.524, "dur": 460.812,
    "args": {
      "External id": 2484144,"Record function id": 0, "Sequence number": 24740545, "Fwd thread id": 1, "Ev Idx": 943
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SwiGLULinearFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762180.676, "dur": 443.505,
    "args": {
      "External id": 2484145,"Sequence number": 24740545, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 944
    }
  },
  {
    "ph": "f", "id": 103, "pid": 1336754, "tid": 1381179, "ts": 1514656762180.676,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762213.418, "dur": 7.514,
    "args": {
      "External id": 2484146,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 945
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_reshape_alias", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762217.633, "dur": 3.040,
    "args": {
      "External id": 2484147,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]", "[8192, 1]"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList"], "Input Strides": [[33554432, 8192, 1], [], []], "Input Dims": [[16, 4096, 2048], [], []], "Ev Idx": 946
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762223.938, "dur": 9.411,
    "args": {
      "External id": 2484148,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 947
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762225.309, "dur": 7.140,
    "args": {
      "External id": 2484149,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 948
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762228.755, "dur": 3.480,
    "args": {
      "External id": 2484150,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 949
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762236.655, "dur": 105.531,
    "args": {
      "External id": 2484151,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8192, 1], [1, 5632], []], "Input Dims": [[65536, 2048], [5632, 2048], []], "Ev Idx": 950
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762237.702, "dur": 4.366,
    "args": {
      "External id": 2484152,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 5632]], "Input Dims": [[5632, 2048]], "Ev Idx": 951
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762240.228, "dur": 1.351,
    "args": {
      "External id": 2484153,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 5632], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 952
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762241.065, "dur": 0.383,
    "args": {
      "External id": 2484154,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[5632, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 5632], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 953
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762243.399, "dur": 98.355,
    "args": {
      "External id": 2484155,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8192, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 954
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762244.893, "dur": 95.938,
    "args": {
      "External id": 2484156,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8192, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 955
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762346.046, "dur": 2.072,
    "args": {
      "External id": 2484157,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [23068672, 5632, 1]], "Input Dims": [[65536, 5632], [16, 4096, 5632]], "Ev Idx": 956
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762346.863, "dur": 1.155,
    "args": {
      "External id": 2484158,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 957
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762381.486, "dur": 5.201,
    "args": {
      "External id": 2484159,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 958
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762390.101, "dur": 2.194,
    "args": {
      "External id": 2484160,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 959
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762392.892, "dur": 1.966,
    "args": {
      "External id": 2484161,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 960
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762428.514, "dur": 1.878,
    "args": {
      "External id": 2484162,"Record function id": 0, "Concrete Inputs": ["", "[-1, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 961
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762429.098, "dur": 1.149,
    "args": {
      "External id": 2484163,"Record function id": 0, "Concrete Inputs": ["", "[-1, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 962
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::einsum", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762452.002, "dur": 152.819,
    "args": {
      "External id": 2484164,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["", "TensorList", ""], "Input Strides": [[], [[8192, 1], [5632, 1]], []], "Input Dims": [[], [[65536, 2048], [65536, 5632]], []], "Ev Idx": 963
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762456.872, "dur": 7.753,
    "args": {
      "External id": 2484165,"Record function id": 0, "Concrete Inputs": ["", "2"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 964
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762461.404, "dur": 2.393,
    "args": {
      "External id": 2484166,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048, 1]", "[8192, 1, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[8192, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 965
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762466.013, "dur": 7.787,
    "args": {
      "External id": 2484167,"Record function id": 0, "Concrete Inputs": ["", "[1, 2, 0]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8192, 1, 1], []], "Input Dims": [[65536, 2048, 1], []], "Ev Idx": 966
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762472.642, "dur": 0.475,
    "args": {
      "External id": 2484168,"Record function id": 0, "Concrete Inputs": ["", "[2048, 1, 65536]", "[1, 1, 8192]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[65536, 2048, 1], [], [], []], "Ev Idx": 967
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762474.992, "dur": 1.617,
    "args": {
      "External id": 2484169,"Record function id": 0, "Concrete Inputs": ["", "2"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 968
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762475.651, "dur": 0.609,
    "args": {
      "External id": 2484170,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632, 1]", "[5632, 1, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[65536, 5632], [], [], []], "Ev Idx": 969
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762479.242, "dur": 2.029,
    "args": {
      "External id": 2484171,"Record function id": 0, "Concrete Inputs": ["", "[2, 1, 0]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 1], []], "Input Dims": [[65536, 5632, 1], []], "Ev Idx": 970
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762479.957, "dur": 0.613,
    "args": {
      "External id": 2484172,"Record function id": 0, "Concrete Inputs": ["", "[1, 5632, 65536]", "[1, 1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1, 1], [], [], []], "Input Dims": [[65536, 5632, 1], [], [], []], "Ev Idx": 971
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762487.014, "dur": 1.585,
    "args": {
      "External id": 2484173,"Record function id": 0, "Concrete Inputs": ["", "[0, 2, 1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1, 1, 8192], []], "Input Dims": [[2048, 1, 65536], []], "Ev Idx": 972
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762487.823, "dur": 0.489,
    "args": {
      "External id": 2484174,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536, 1]", "[1, 8192, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1, 8192], [], [], []], "Input Dims": [[2048, 1, 65536], [], [], []], "Ev Idx": 973
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762489.531, "dur": 6.787,
    "args": {
      "External id": 2484175,"Record function id": 0, "Concrete Inputs": ["", "[1, 2048, 65536]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1, 8192, 1], []], "Input Dims": [[2048, 65536, 1], []], "Ev Idx": 974
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_reshape_alias", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762494.165, "dur": 1.992,
    "args": {
      "External id": 2484176,"Record function id": 0, "Concrete Inputs": ["", "[1, 2048, 65536]", "[2048, 1, 8192]"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList"], "Input Strides": [[1, 8192, 1], [], []], "Input Dims": [[2048, 65536, 1], [], []], "Ev Idx": 975
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762496.890, "dur": 3.459,
    "args": {
      "External id": 2484177,"Record function id": 0, "Concrete Inputs": ["", "[2, 1, 0]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1, 1, 5632], []], "Input Dims": [[1, 5632, 65536], []], "Ev Idx": 976
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762499.563, "dur": 0.476,
    "args": {
      "External id": 2484178,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632, 1]", "[5632, 1, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1, 5632], [], [], []], "Input Dims": [[1, 5632, 65536], [], [], []], "Ev Idx": 977
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762503.210, "dur": 3.672,
    "args": {
      "External id": 2484179,"Record function id": 0, "Concrete Inputs": ["", "[1, 65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 1], []], "Input Dims": [[65536, 5632, 1], []], "Ev Idx": 978
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762503.592, "dur": 3.198,
    "args": {
      "External id": 2484180,"Record function id": 0, "Concrete Inputs": ["", "[1, 65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 1], []], "Input Dims": [[65536, 5632, 1], []], "Ev Idx": 979
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762508.317, "dur": 80.560,
    "args": {
      "External id": 2484181,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1, 8192], [369098752, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632]], "Ev Idx": 980
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762593.985, "dur": 1.906,
    "args": {
      "External id": 2484182,"Record function id": 0, "Concrete Inputs": ["", "[2048, 1, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[11534336, 5632, 1], []], "Input Dims": [[1, 2048, 5632], []], "Ev Idx": 981
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762596.509, "dur": 4.579,
    "args": {
      "External id": 2484183,"Record function id": 0, "Concrete Inputs": ["", "[0, 2, 1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 5632, 1], []], "Input Dims": [[2048, 1, 5632], []], "Ev Idx": 982
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762599.851, "dur": 0.557,
    "args": {
      "External id": 2484184,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632, 1]", "[5632, 1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 5632, 1], [], [], []], "Input Dims": [[2048, 1, 5632], [], [], []], "Ev Idx": 983
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762603.052, "dur": 0.890,
    "args": {
      "External id": 2484185,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 5632], []], "Input Dims": [[2048, 5632, 1], []], "Ev Idx": 984
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762651.759, "dur": 9.386,
    "args": {
      "External id": 2484186,"Record function id": 0, "Ev Idx": 985
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762653.985, "dur": 6.527,
    "args": {
      "External id": 2484187,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 986
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762656.246, "dur": 3.378,
    "args": {
      "External id": 2484188,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 987
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762657.121, "dur": 2.413,
    "args": {
      "External id": 2484189,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 988
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762665.165, "dur": 8.500,
    "args": {
      "External id": 2484190,"Record function id": 0, "Sequence number": 24740544, "Fwd thread id": 1, "Ev Idx": 989
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762665.890, "dur": 5.446,
    "args": {
      "External id": 2484191,"Sequence number": 24740544, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[23068672, 5632, 1]], "Input Dims": [[16, 4096, 5632]], "Ev Idx": 990
    }
  },
  {
    "ph": "f", "id": 104, "pid": 1336754, "tid": 1381179, "ts": 1514656762665.890,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762669.509, "dur": 1.648,
    "args": {
      "External id": 2484192,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 991
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762670.074, "dur": 0.930,
    "args": {
      "External id": 2484193,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 992
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762677.283, "dur": 132.639,
    "args": {
      "External id": 2484194,"Record function id": 0, "Sequence number": 24740543, "Fwd thread id": 1, "Ev Idx": 993
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762678.352, "dur": 123.754,
    "args": {
      "External id": 2484195,"Sequence number": 24740543, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 994
    }
  },
  {
    "ph": "f", "id": 105, "pid": 1336754, "tid": 1381179, "ts": 1514656762678.352,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762682.815, "dur": 6.216,
    "args": {
      "External id": 2484196,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 995
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762684.159, "dur": 4.315,
    "args": {
      "External id": 2484197,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[65536, 5632], [], []], "Ev Idx": 996
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762687.464, "dur": 0.848,
    "args": {
      "External id": 2484198,"Record function id": 0, "Concrete Inputs": ["", "[5632, 65536]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[65536, 5632], [], [], []], "Ev Idx": 997
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762690.192, "dur": 45.220,
    "args": {
      "External id": 2484199,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048]], "Ev Idx": 998
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762736.786, "dur": 5.964,
    "args": {
      "External id": 2484200,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 999
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762737.343, "dur": 4.573,
    "args": {
      "External id": 2484201,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 1000
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762740.740, "dur": 0.983,
    "args": {
      "External id": 2484202,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 1001
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762744.471, "dur": 7.996,
    "args": {
      "External id": 2484203,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 1002
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762745.687, "dur": 6.344,
    "args": {
      "External id": 2484204,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 1003
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762749.107, "dur": 2.860,
    "args": {
      "External id": 2484205,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 1004
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762752.934, "dur": 48.465,
    "args": {
      "External id": 2484206,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048]], "Ev Idx": 1005
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762814.700, "dur": 8.666,
    "args": {
      "External id": 2484207,"Record function id": 0, "Sequence number": 24740542, "Fwd thread id": 1, "Ev Idx": 1006
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762815.972, "dur": 5.961,
    "args": {
      "External id": 2484208,"Sequence number": 24740542, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1007
    }
  },
  {
    "ph": "f", "id": 106, "pid": 1336754, "tid": 1381179, "ts": 1514656762815.972,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762817.550, "dur": 4.234,
    "args": {
      "External id": 2484209,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1008
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762820.531, "dur": 1.122,
    "args": {
      "External id": 2484210,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1009
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762826.662, "dur": 9.733,
    "args": {
      "External id": 2484211,"Record function id": 0, "Sequence number": 24740541, "Fwd thread id": 1, "Ev Idx": 1010
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762827.493, "dur": 6.333,
    "args": {
      "External id": 2484212,"Sequence number": 24740541, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 1011
    }
  },
  {
    "ph": "f", "id": 107, "pid": 1336754, "tid": 1381179, "ts": 1514656762827.493,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762828.630, "dur": 4.996,
    "args": {
      "External id": 2484213,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 1012
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762829.347, "dur": 3.839,
    "args": {
      "External id": 2484214,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 1013
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762832.391, "dur": 0.652,
    "args": {
      "External id": 2484215,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 1014
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762840.194, "dur": 4.476,
    "args": {
      "External id": 2484216,"Record function id": 0, "Ev Idx": 1015
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762841.478, "dur": 2.679,
    "args": {
      "External id": 2484217,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1016
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762842.495, "dur": 1.405,
    "args": {
      "External id": 2484218,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1017
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762842.889, "dur": 0.926,
    "args": {
      "External id": 2484219,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1018
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762850.815, "dur": 8.333,
    "args": {
      "External id": 2484220,"Record function id": 0, "Sequence number": 24740540, "Fwd thread id": 1, "Ev Idx": 1019
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762851.951, "dur": 4.473,
    "args": {
      "External id": 2484221,"Sequence number": 24740540, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[23068672, 5632, 1]], "Input Dims": [[16, 4096, 5632]], "Ev Idx": 1020
    }
  },
  {
    "ph": "f", "id": 108, "pid": 1336754, "tid": 1381179, "ts": 1514656762851.951,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762853.058, "dur": 3.225,
    "args": {
      "External id": 2484222,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 1021
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762855.482, "dur": 0.662,
    "args": {
      "External id": 2484223,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 1022
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762862.359, "dur": 174.855,
    "args": {
      "External id": 2484224,"Record function id": 0, "Sequence number": 24740539, "Fwd thread id": 1, "Ev Idx": 1023
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762885.042, "dur": 141.367,
    "args": {
      "External id": 2484225,"Sequence number": 24740539, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 1024
    }
  },
  {
    "ph": "f", "id": 109, "pid": 1336754, "tid": 1381179, "ts": 1514656762885.042,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762889.884, "dur": 2.082,
    "args": {
      "External id": 2484226,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 1025
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762890.370, "dur": 1.186,
    "args": {
      "External id": 2484227,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[65536, 5632], [], []], "Ev Idx": 1026
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762891.036, "dur": 0.409,
    "args": {
      "External id": 2484228,"Record function id": 0, "Concrete Inputs": ["", "[5632, 65536]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[65536, 5632], [], [], []], "Ev Idx": 1027
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762892.599, "dur": 33.025,
    "args": {
      "External id": 2484229,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048]], "Ev Idx": 1028
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762926.684, "dur": 8.254,
    "args": {
      "External id": 2484230,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1029
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762927.230, "dur": 7.190,
    "args": {
      "External id": 2484231,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 1030
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762929.929, "dur": 4.321,
    "args": {
      "External id": 2484232,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 1031
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762936.186, "dur": 5.639,
    "args": {
      "External id": 2484233,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 1032
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762938.113, "dur": 3.183,
    "args": {
      "External id": 2484234,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 1033
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762940.732, "dur": 0.463,
    "args": {
      "External id": 2484235,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 1034
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656762942.278, "dur": 82.674,
    "args": {
      "External id": 2484236,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048]], "Ev Idx": 1035
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763044.849, "dur": 37.863,
    "args": {
      "External id": 2484237,"Record function id": 0, "Sequence number": 24740538, "Fwd thread id": 1, "Ev Idx": 1036
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763046.202, "dur": 6.647,
    "args": {
      "External id": 2484238,"Sequence number": 24740538, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1037
    }
  },
  {
    "ph": "f", "id": 110, "pid": 1336754, "tid": 1381179, "ts": 1514656763046.202,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763050.505, "dur": 2.181,
    "args": {
      "External id": 2484239,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1038
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763051.049, "dur": 1.525,
    "args": {
      "External id": 2484240,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1039
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763055.555, "dur": 24.095,
    "args": {
      "External id": 2484241,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 1040
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763086.478, "dur": 10.031,
    "args": {
      "External id": 2484242,"Record function id": 0, "Sequence number": 24740537, "Fwd thread id": 1, "Ev Idx": 1041
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763087.537, "dur": 6.335,
    "args": {
      "External id": 2484243,"Sequence number": 24740537, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 1042
    }
  },
  {
    "ph": "f", "id": 111, "pid": 1336754, "tid": 1381179, "ts": 1514656763087.537,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763088.425, "dur": 5.252,
    "args": {
      "External id": 2484244,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 1043
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763089.347, "dur": 3.715,
    "args": {
      "External id": 2484245,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 1044
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763092.349, "dur": 0.593,
    "args": {
      "External id": 2484246,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 1045
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763100.663, "dur": 8.527,
    "args": {
      "External id": 2484247,"Record function id": 0, "Ev Idx": 1046
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763102.038, "dur": 6.624,
    "args": {
      "External id": 2484248,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1047
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763103.345, "dur": 4.831,
    "args": {
      "External id": 2484249,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1048
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763106.493, "dur": 1.573,
    "args": {
      "External id": 2484250,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1049
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: LayerNormFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763113.263, "dur": 409.410,
    "args": {
      "External id": 2484251,"Record function id": 0, "Sequence number": 24740536, "Fwd thread id": 1, "Ev Idx": 1050
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763114.510, "dur": 373.763,
    "args": {
      "External id": 2484252,"Sequence number": 24740536, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [33554432, 8192, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 1051
    }
  },
  {
    "ph": "f", "id": 112, "pid": 1336754, "tid": 1381179, "ts": 1514656763114.510,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::contiguous", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763139.369, "dur": 33.110,
    "args": {
      "External id": 2484253,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1052
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763140.620, "dur": 31.654,
    "args": {
      "External id": 2484254,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1053
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763143.104, "dur": 5.909,
    "args": {
      "External id": 2484255,"Record function id": 0, "Concrete Inputs": ["", "15", "0", "", "", "0"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[33554432, 8192, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], [], []], "Ev Idx": 1054
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763145.526, "dur": 2.887,
    "args": {
      "External id": 2484256,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 2048]", "15", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1055
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763150.189, "dur": 21.588,
    "args": {
      "External id": 2484257,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 1056
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763186.931, "dur": 4.077,
    "args": {
      "External id": 2484258,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1057
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763187.783, "dur": 3.103,
    "args": {
      "External id": 2484259,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1058
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763194.362, "dur": 3.462,
    "args": {
      "External id": 2484260,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1059
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763196.935, "dur": 0.780,
    "args": {
      "External id": 2484261,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1060
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763209.617, "dur": 2.518,
    "args": {
      "External id": 2484262,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1061
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763222.965, "dur": 1.866,
    "args": {
      "External id": 2484263,"Record function id": 0, "Concrete Inputs": ["[132, 2048]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1062
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763377.664, "dur": 2.607,
    "args": {
      "External id": 2484264,"Record function id": 0, "Concrete Inputs": ["", "[1, -1, 2048]"], "Input type": ["float", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[132, 2048], []], "Ev Idx": 1063
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763385.920, "dur": 31.573,
    "args": {
      "External id": 2484265,"Record function id": 0, "Concrete Inputs": ["", "[1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[270336, 2048, 1], [], [], []], "Input Dims": [[1, 132, 2048], [], [], []], "Ev Idx": 1064
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763394.399, "dur": 0.675,
    "args": {
      "External id": 2484266,"Record function id": 0, "Concrete Inputs": ["", "[1, 1, 2048]", "[2048, 0, 1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[1, 2048], [], [], []], "Ev Idx": 1065
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763422.714, "dur": 31.724,
    "args": {
      "External id": 2484267,"Record function id": 0, "Concrete Inputs": ["", "", "15", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], []], "Ev Idx": 1066
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763426.538, "dur": 27.651,
    "args": {
      "External id": 2484268,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], [], []], "Ev Idx": 1067
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763432.350, "dur": 4.486,
    "args": {
      "External id": 2484269,"Record function id": 0, "Concrete Inputs": ["[1, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1068
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763437.969, "dur": 15.728,
    "args": {
      "External id": 2484270,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "float", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[1, 2048], [1, 2048], []], "Ev Idx": 1069
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763458.542, "dur": 2.195,
    "args": {
      "External id": 2484271,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1]], "Input Dims": [[1, 2048], [2048]], "Ev Idx": 1070
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763459.542, "dur": 1.096,
    "args": {
      "External id": 2484272,"Record function id": 0, "Concrete Inputs": ["", "[2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[1, 2048], []], "Ev Idx": 1071
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763467.077, "dur": 6.465,
    "args": {
      "External id": 2484273,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1072
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763470.125, "dur": 3.308,
    "args": {
      "External id": 2484274,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1073
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763475.629, "dur": 4.195,
    "args": {
      "External id": 2484275,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1074
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763478.721, "dur": 1.020,
    "args": {
      "External id": 2484276,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1075
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763503.502, "dur": 17.458,
    "args": {
      "External id": 2484277,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 1076
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763550.474, "dur": 10.064,
    "args": {
      "External id": 2484278,"Record function id": 0, "Ev Idx": 1077
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763553.138, "dur": 6.584,
    "args": {
      "External id": 2484279,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1078
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763554.947, "dur": 3.430,
    "args": {
      "External id": 2484280,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1079
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763555.824, "dur": 2.327,
    "args": {
      "External id": 2484281,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1080
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763564.882, "dur": 11.043,
    "args": {
      "External id": 2484282,"Record function id": 0, "Sequence number": 24740535, "Fwd thread id": 1, "Ev Idx": 1081
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763565.743, "dur": 6.492,
    "args": {
      "External id": 2484283,"Sequence number": 24740535, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1082
    }
  },
  {
    "ph": "f", "id": 113, "pid": 1336754, "tid": 1381179, "ts": 1514656763565.743,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763569.671, "dur": 2.385,
    "args": {
      "External id": 2484284,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1083
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763570.296, "dur": 1.613,
    "args": {
      "External id": 2484285,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1084
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763579.602, "dur": 141.914,
    "args": {
      "External id": 2484286,"Record function id": 0, "Sequence number": 24740534, "Fwd thread id": 1, "Ev Idx": 1085
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763580.569, "dur": 133.848,
    "args": {
      "External id": 2484287,"Sequence number": 24740534, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1086
    }
  },
  {
    "ph": "f", "id": 114, "pid": 1336754, "tid": 1381179, "ts": 1514656763580.569,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763584.712, "dur": 6.657,
    "args": {
      "External id": 2484288,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1087
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763585.996, "dur": 4.826,
    "args": {
      "External id": 2484289,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 1088
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763589.680, "dur": 0.984,
    "args": {
      "External id": 2484290,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 1089
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763592.700, "dur": 60.004,
    "args": {
      "External id": 2484291,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 1090
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763653.625, "dur": 8.680,
    "args": {
      "External id": 2484292,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1091
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763654.352, "dur": 7.386,
    "args": {
      "External id": 2484293,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1092
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763659.109, "dur": 2.474,
    "args": {
      "External id": 2484294,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1093
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763664.331, "dur": 5.890,
    "args": {
      "External id": 2484295,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1094
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763665.801, "dur": 3.946,
    "args": {
      "External id": 2484296,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1095
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763669.045, "dur": 0.637,
    "args": {
      "External id": 2484297,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1096
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763670.637, "dur": 42.932,
    "args": {
      "External id": 2484298,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 1097
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763726.718, "dur": 11.610,
    "args": {
      "External id": 2484299,"Record function id": 0, "Sequence number": 24740533, "Fwd thread id": 1, "Ev Idx": 1098
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763727.927, "dur": 7.253,
    "args": {
      "External id": 2484300,"Sequence number": 24740533, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1099
    }
  },
  {
    "ph": "f", "id": 115, "pid": 1336754, "tid": 1381179, "ts": 1514656763727.927,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763729.118, "dur": 5.904,
    "args": {
      "External id": 2484301,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1100
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763731.723, "dur": 3.147,
    "args": {
      "External id": 2484302,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1101
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763744.326, "dur": 9.268,
    "args": {
      "External id": 2484303,"Record function id": 0, "Sequence number": 24740532, "Fwd thread id": 1, "Ev Idx": 1102
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763745.408, "dur": 6.068,
    "args": {
      "External id": 2484304,"Sequence number": 24740532, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1103
    }
  },
  {
    "ph": "f", "id": 116, "pid": 1336754, "tid": 1381179, "ts": 1514656763745.408,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763746.250, "dur": 4.996,
    "args": {
      "External id": 2484305,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1104
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763747.014, "dur": 3.766,
    "args": {
      "External id": 2484306,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1105
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763749.981, "dur": 0.641,
    "args": {
      "External id": 2484307,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1106
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763757.428, "dur": 4.851,
    "args": {
      "External id": 2484308,"Record function id": 0, "Ev Idx": 1107
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763758.879, "dur": 2.917,
    "args": {
      "External id": 2484309,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1108
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763759.814, "dur": 1.740,
    "args": {
      "External id": 2484310,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1109
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763760.418, "dur": 1.008,
    "args": {
      "External id": 2484311,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1110
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763768.070, "dur": 6.377,
    "args": {
      "External id": 2484312,"Record function id": 0, "Sequence number": 24740531, "Fwd thread id": 1, "Ev Idx": 1111
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763768.753, "dur": 3.390,
    "args": {
      "External id": 2484313,"Sequence number": 24740531, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1112
    }
  },
  {
    "ph": "f", "id": 117, "pid": 1336754, "tid": 1381179, "ts": 1514656763768.753,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763770.307, "dur": 1.659,
    "args": {
      "External id": 2484314,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1113
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763770.698, "dur": 1.135,
    "args": {
      "External id": 2484315,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1114
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: FlashAttnFuncBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763780.293, "dur": 344.262,
    "args": {
      "External id": 2484316,"Record function id": 0, "Sequence number": 24740530, "Fwd thread id": 1, "Ev Idx": 1115
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FlashAttnFuncBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763781.615, "dur": 318.266,
    "args": {
      "External id": 2484317,"Sequence number": 24740530, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1116
    }
  },
  {
    "ph": "f", "id": 118, "pid": 1336754, "tid": 1381179, "ts": 1514656763781.615,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763799.359, "dur": 5.573,
    "args": {
      "External id": 2484318,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 1117
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763801.188, "dur": 3.301,
    "args": {
      "External id": 2484319,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1118
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763806.731, "dur": 7.286,
    "args": {
      "External id": 2484320,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 1119
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763809.671, "dur": 4.121,
    "args": {
      "External id": 2484321,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1120
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763823.530, "dur": 5.229,
    "args": {
      "External id": 2484322,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 1121
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763826.590, "dur": 1.979,
    "args": {
      "External id": 2484323,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1122
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763857.164, "dur": 215.164,
    "args": {
      "External id": 2484324,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 1123
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763936.046, "dur": 5.517,
    "args": {
      "External id": 2484325,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1124
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656763943.232, "dur": 3.914,
    "args": {
      "External id": 2484326,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1125
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764086.902, "dur": 3.997,
    "args": {
      "External id": 2484327,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1126
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764094.007, "dur": 0.658,
    "args": {
      "External id": 2484328,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1127
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764096.523, "dur": 0.656,
    "args": {
      "External id": 2484329,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1128
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RotaryEmbeddingFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764133.520, "dur": 237.504,
    "args": {
      "External id": 2484330,"Record function id": 0, "Sequence number": 24740529, "Fwd thread id": 1, "Ev Idx": 1129
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764135.265, "dur": 227.312,
    "args": {
      "External id": 2484331,"Sequence number": 24740529, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1130
    }
  },
  {
    "ph": "f", "id": 119, "pid": 1336754, "tid": 1381179, "ts": 1514656764135.265,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764156.687, "dur": 48.032,
    "args": {
      "External id": 2484332,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1131
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764159.764, "dur": 3.754,
    "args": {
      "External id": 2484333,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1132
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764164.895, "dur": 39.194,
    "args": {
      "External id": 2484334,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], []], "Ev Idx": 1133
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764214.709, "dur": 6.142,
    "args": {
      "External id": 2484335,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 1134
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764218.140, "dur": 2.377,
    "args": {
      "External id": 2484336,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1135
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RotaryEmbeddingFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764378.963, "dur": 190.077,
    "args": {
      "External id": 2484337,"Record function id": 0, "Sequence number": 24740528, "Fwd thread id": 1, "Ev Idx": 1136
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764380.434, "dur": 180.681,
    "args": {
      "External id": 2484338,"Sequence number": 24740528, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1137
    }
  },
  {
    "ph": "f", "id": 120, "pid": 1336754, "tid": 1381179, "ts": 1514656764380.434,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764393.115, "dur": 44.705,
    "args": {
      "External id": 2484339,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1138
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764395.065, "dur": 2.936,
    "args": {
      "External id": 2484340,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1139
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764403.596, "dur": 33.704,
    "args": {
      "External id": 2484341,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], []], "Ev Idx": 1140
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764444.842, "dur": 5.854,
    "args": {
      "External id": 2484342,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 1141
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764448.107, "dur": 2.214,
    "args": {
      "External id": 2484343,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1142
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764578.042, "dur": 20.917,
    "args": {
      "External id": 2484344,"Record function id": 0, "Sequence number": 24740527, "Fwd thread id": 1, "Ev Idx": 1143
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764579.806, "dur": 15.667,
    "args": {
      "External id": 2484345,"Sequence number": 24740527, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1144
    }
  },
  {
    "ph": "f", "id": 121, "pid": 1336754, "tid": 1381179, "ts": 1514656764579.806,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764582.551, "dur": 12.594,
    "args": {
      "External id": 2484346,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1145
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764586.208, "dur": 8.694,
    "args": {
      "External id": 2484347,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1146
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764603.007, "dur": 8.955,
    "args": {
      "External id": 2484348,"Record function id": 0, "Sequence number": 24740526, "Fwd thread id": 1, "Ev Idx": 1147
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764604.149, "dur": 4.802,
    "args": {
      "External id": 2484349,"Sequence number": 24740526, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1148
    }
  },
  {
    "ph": "f", "id": 122, "pid": 1336754, "tid": 1381179, "ts": 1514656764604.149,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764605.332, "dur": 3.486,
    "args": {
      "External id": 2484350,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1149
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764607.821, "dur": 0.875,
    "args": {
      "External id": 2484351,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1150
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764614.986, "dur": 6.486,
    "args": {
      "External id": 2484352,"Record function id": 0, "Sequence number": 24740525, "Fwd thread id": 1, "Ev Idx": 1151
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764615.657, "dur": 3.102,
    "args": {
      "External id": 2484353,"Sequence number": 24740525, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1152
    }
  },
  {
    "ph": "f", "id": 123, "pid": 1336754, "tid": 1381179, "ts": 1514656764615.657,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764617.058, "dur": 1.562,
    "args": {
      "External id": 2484354,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1153
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764617.651, "dur": 0.880,
    "args": {
      "External id": 2484355,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1154
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764624.914, "dur": 11.072,
    "args": {
      "External id": 2484356,"Record function id": 0, "Sequence number": 24740524, "Fwd thread id": 1, "Ev Idx": 1155
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764626.032, "dur": 6.732,
    "args": {
      "External id": 2484357,"Sequence number": 24740524, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1156
    }
  },
  {
    "ph": "f", "id": 124, "pid": 1336754, "tid": 1381179, "ts": 1514656764626.032,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764627.225, "dur": 5.407,
    "args": {
      "External id": 2484358,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1157
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764631.759, "dur": 0.780,
    "args": {
      "External id": 2484359,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1158
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764639.457, "dur": 161.921,
    "args": {
      "External id": 2484360,"Record function id": 0, "Sequence number": 24740523, "Fwd thread id": 1, "Ev Idx": 1159
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764640.469, "dur": 153.896,
    "args": {
      "External id": 2484361,"Sequence number": 24740523, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1160
    }
  },
  {
    "ph": "f", "id": 125, "pid": 1336754, "tid": 1381179, "ts": 1514656764640.469,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764644.924, "dur": 6.484,
    "args": {
      "External id": 2484362,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1161
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764646.762, "dur": 3.961,
    "args": {
      "External id": 2484363,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 1162
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764648.864, "dur": 1.588,
    "args": {
      "External id": 2484364,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 1163
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764655.845, "dur": 72.381,
    "args": {
      "External id": 2484365,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 1164
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764729.641, "dur": 4.649,
    "args": {
      "External id": 2484366,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1165
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764730.491, "dur": 2.871,
    "args": {
      "External id": 2484367,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1166
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764732.029, "dur": 1.154,
    "args": {
      "External id": 2484368,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1167
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764736.497, "dur": 6.424,
    "args": {
      "External id": 2484369,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1168
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764738.137, "dur": 4.177,
    "args": {
      "External id": 2484370,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1169
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764741.573, "dur": 0.669,
    "args": {
      "External id": 2484371,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1170
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764745.827, "dur": 47.778,
    "args": {
      "External id": 2484372,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 1171
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764808.410, "dur": 6.450,
    "args": {
      "External id": 2484373,"Record function id": 0, "Sequence number": 24740522, "Fwd thread id": 1, "Ev Idx": 1172
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764809.330, "dur": 3.269,
    "args": {
      "External id": 2484374,"Sequence number": 24740522, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1173
    }
  },
  {
    "ph": "f", "id": 126, "pid": 1336754, "tid": 1381179, "ts": 1514656764809.330,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764810.668, "dur": 1.773,
    "args": {
      "External id": 2484375,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1174
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764811.271, "dur": 1.063,
    "args": {
      "External id": 2484376,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1175
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764818.050, "dur": 9.451,
    "args": {
      "External id": 2484377,"Record function id": 0, "Sequence number": 24740521, "Fwd thread id": 1, "Ev Idx": 1176
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764818.855, "dur": 5.580,
    "args": {
      "External id": 2484378,"Sequence number": 24740521, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1177
    }
  },
  {
    "ph": "f", "id": 127, "pid": 1336754, "tid": 1381179, "ts": 1514656764818.855,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764819.849, "dur": 4.359,
    "args": {
      "External id": 2484379,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1178
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764820.460, "dur": 3.287,
    "args": {
      "External id": 2484380,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1179
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764823.209, "dur": 0.432,
    "args": {
      "External id": 2484381,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1180
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764835.734, "dur": 10.283,
    "args": {
      "External id": 2484382,"Record function id": 0, "Ev Idx": 1181
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764837.306, "dur": 7.965,
    "args": {
      "External id": 2484383,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1182
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764840.761, "dur": 4.002,
    "args": {
      "External id": 2484384,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1183
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764841.979, "dur": 2.633,
    "args": {
      "External id": 2484385,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1184
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764849.205, "dur": 6.628,
    "args": {
      "External id": 2484386,"Record function id": 0, "Sequence number": 24740520, "Fwd thread id": 1, "Ev Idx": 1185
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764850.345, "dur": 2.915,
    "args": {
      "External id": 2484387,"Sequence number": 24740520, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1186
    }
  },
  {
    "ph": "f", "id": 128, "pid": 1336754, "tid": 1381179, "ts": 1514656764850.345,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764851.595, "dur": 1.520,
    "args": {
      "External id": 2484388,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1187
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764852.224, "dur": 0.775,
    "args": {
      "External id": 2484389,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1188
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764858.866, "dur": 96.688,
    "args": {
      "External id": 2484390,"Record function id": 0, "Sequence number": 24740519, "Fwd thread id": 1, "Ev Idx": 1189
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764859.678, "dur": 88.719,
    "args": {
      "External id": 2484391,"Sequence number": 24740519, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1190
    }
  },
  {
    "ph": "f", "id": 129, "pid": 1336754, "tid": 1381179, "ts": 1514656764859.678,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764864.585, "dur": 2.388,
    "args": {
      "External id": 2484392,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1191
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764865.094, "dur": 1.455,
    "args": {
      "External id": 2484393,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 1192
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764865.871, "dur": 0.571,
    "args": {
      "External id": 2484394,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 1193
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764867.607, "dur": 28.124,
    "args": {
      "External id": 2484395,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 1194
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764896.742, "dur": 6.811,
    "args": {
      "External id": 2484396,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1195
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764897.285, "dur": 5.727,
    "args": {
      "External id": 2484397,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1196
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764902.160, "dur": 0.724,
    "args": {
      "External id": 2484398,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1197
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764904.631, "dur": 6.135,
    "args": {
      "External id": 2484399,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1198
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764906.493, "dur": 3.746,
    "args": {
      "External id": 2484400,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1199
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764907.449, "dur": 2.722,
    "args": {
      "External id": 2484401,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1200
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764911.493, "dur": 36.195,
    "args": {
      "External id": 2484402,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 1201
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764960.051, "dur": 76.945,
    "args": {
      "External id": 2484403,"Record function id": 0, "Sequence number": 24740518, "Fwd thread id": 1, "Ev Idx": 1202
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764961.118, "dur": 7.600,
    "args": {
      "External id": 2484404,"Sequence number": 24740518, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1203
    }
  },
  {
    "ph": "f", "id": 130, "pid": 1336754, "tid": 1381179, "ts": 1514656764961.118,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764965.057, "dur": 3.514,
    "args": {
      "External id": 2484405,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1204
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764967.368, "dur": 1.106,
    "args": {
      "External id": 2484406,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1205
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336754, "tid": 1381179,
    "ts": 1514656764971.708, "dur": 61.864,
    "args": {
      "External id": 2484407,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 1206
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765045.468, "dur": 8.910,
    "args": {
      "External id": 2484408,"Record function id": 0, "Sequence number": 24740517, "Fwd thread id": 1, "Ev Idx": 1207
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765046.658, "dur": 5.368,
    "args": {
      "External id": 2484409,"Sequence number": 24740517, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1208
    }
  },
  {
    "ph": "f", "id": 131, "pid": 1336754, "tid": 1381179, "ts": 1514656765046.658,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765047.834, "dur": 3.954,
    "args": {
      "External id": 2484410,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1209
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765048.969, "dur": 2.095,
    "args": {
      "External id": 2484411,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1210
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765050.272, "dur": 0.658,
    "args": {
      "External id": 2484412,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1211
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765058.646, "dur": 7.893,
    "args": {
      "External id": 2484413,"Record function id": 0, "Ev Idx": 1212
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765059.825, "dur": 6.200,
    "args": {
      "External id": 2484414,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1213
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765061.316, "dur": 4.195,
    "args": {
      "External id": 2484415,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1214
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765063.883, "dur": 1.540,
    "args": {
      "External id": 2484416,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1215
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765069.881, "dur": 10.983,
    "args": {
      "External id": 2484417,"Record function id": 0, "Sequence number": 24740516, "Fwd thread id": 1, "Ev Idx": 1216
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765070.789, "dur": 7.820,
    "args": {
      "External id": 2484418,"Sequence number": 24740516, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1217
    }
  },
  {
    "ph": "f", "id": 132, "pid": 1336754, "tid": 1381179, "ts": 1514656765070.789,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765071.939, "dur": 6.510,
    "args": {
      "External id": 2484419,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1218
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765077.305, "dur": 1.007,
    "args": {
      "External id": 2484420,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1219
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765083.848, "dur": 104.702,
    "args": {
      "External id": 2484421,"Record function id": 0, "Sequence number": 24740515, "Fwd thread id": 1, "Ev Idx": 1220
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765084.765, "dur": 93.289,
    "args": {
      "External id": 2484422,"Sequence number": 24740515, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1221
    }
  },
  {
    "ph": "f", "id": 133, "pid": 1336754, "tid": 1381179, "ts": 1514656765084.765,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765089.628, "dur": 2.167,
    "args": {
      "External id": 2484423,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1222
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765090.136, "dur": 1.180,
    "args": {
      "External id": 2484424,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 1223
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765090.755, "dur": 0.446,
    "args": {
      "External id": 2484425,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 1224
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765092.636, "dur": 35.289,
    "args": {
      "External id": 2484426,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 1225
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765128.740, "dur": 7.426,
    "args": {
      "External id": 2484427,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1226
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765129.551, "dur": 6.114,
    "args": {
      "External id": 2484428,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1227
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765132.699, "dur": 2.736,
    "args": {
      "External id": 2484429,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1228
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765137.307, "dur": 5.785,
    "args": {
      "External id": 2484430,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1229
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765139.129, "dur": 3.564,
    "args": {
      "External id": 2484431,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1230
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765142.201, "dur": 0.420,
    "args": {
      "External id": 2484432,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1231
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765143.650, "dur": 33.865,
    "args": {
      "External id": 2484433,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 1232
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765192.765, "dur": 26.735,
    "args": {
      "External id": 2484434,"Record function id": 0, "Sequence number": 24740514, "Fwd thread id": 1, "Ev Idx": 1233
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765193.863, "dur": 3.643,
    "args": {
      "External id": 2484435,"Sequence number": 24740514, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1234
    }
  },
  {
    "ph": "f", "id": 134, "pid": 1336754, "tid": 1381179, "ts": 1514656765193.863,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765195.376, "dur": 1.991,
    "args": {
      "External id": 2484436,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1235
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765195.994, "dur": 1.230,
    "args": {
      "External id": 2484437,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1236
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765200.198, "dur": 17.306,
    "args": {
      "External id": 2484438,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 1237
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765224.559, "dur": 7.279,
    "args": {
      "External id": 2484439,"Record function id": 0, "Sequence number": 24740513, "Fwd thread id": 1, "Ev Idx": 1238
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765225.514, "dur": 3.834,
    "args": {
      "External id": 2484440,"Sequence number": 24740513, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1239
    }
  },
  {
    "ph": "f", "id": 135, "pid": 1336754, "tid": 1381179, "ts": 1514656765225.514,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765226.538, "dur": 2.622,
    "args": {
      "External id": 2484441,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1240
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765227.078, "dur": 1.630,
    "args": {
      "External id": 2484442,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1241
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765227.952, "dur": 0.630,
    "args": {
      "External id": 2484443,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1242
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765237.753, "dur": 5.131,
    "args": {
      "External id": 2484444,"Record function id": 0, "Ev Idx": 1243
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765238.995, "dur": 3.443,
    "args": {
      "External id": 2484445,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1244
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765239.973, "dur": 1.968,
    "args": {
      "External id": 2484446,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1245
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765240.655, "dur": 1.210,
    "args": {
      "External id": 2484447,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1246
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: LayerNormFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765249.047, "dur": 375.989,
    "args": {
      "External id": 2484448,"Record function id": 0, "Sequence number": 24740512, "Fwd thread id": 1, "Ev Idx": 1247
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765250.273, "dur": 341.831,
    "args": {
      "External id": 2484449,"Sequence number": 24740512, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1248
    }
  },
  {
    "ph": "f", "id": 136, "pid": 1336754, "tid": 1381179, "ts": 1514656765250.273,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765283.951, "dur": 2.058,
    "args": {
      "External id": 2484450,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1249
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765284.426, "dur": 1.369,
    "args": {
      "External id": 2484451,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1250
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765299.859, "dur": 5.492,
    "args": {
      "External id": 2484452,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1251
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765314.396, "dur": 1.961,
    "args": {
      "External id": 2484453,"Record function id": 0, "Concrete Inputs": ["[132, 2048]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1252
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765459.945, "dur": 2.038,
    "args": {
      "External id": 2484454,"Record function id": 0, "Concrete Inputs": ["", "[1, -1, 2048]"], "Input type": ["float", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[132, 2048], []], "Ev Idx": 1253
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765465.764, "dur": 33.432,
    "args": {
      "External id": 2484455,"Record function id": 0, "Concrete Inputs": ["", "[1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[270336, 2048, 1], [], [], []], "Input Dims": [[1, 132, 2048], [], [], []], "Ev Idx": 1254
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765474.650, "dur": 0.853,
    "args": {
      "External id": 2484456,"Record function id": 0, "Concrete Inputs": ["", "[1, 1, 2048]", "[2048, 0, 1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[1, 2048], [], [], []], "Ev Idx": 1255
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765504.808, "dur": 59.965,
    "args": {
      "External id": 2484457,"Record function id": 0, "Concrete Inputs": ["", "", "15", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], []], "Ev Idx": 1256
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765509.247, "dur": 55.322,
    "args": {
      "External id": 2484458,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], [], []], "Ev Idx": 1257
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765515.541, "dur": 5.712,
    "args": {
      "External id": 2484459,"Record function id": 0, "Concrete Inputs": ["[1, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1258
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765523.076, "dur": 40.703,
    "args": {
      "External id": 2484460,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "float", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[1, 2048], [1, 2048], []], "Ev Idx": 1259
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765570.968, "dur": 3.054,
    "args": {
      "External id": 2484461,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1]], "Input Dims": [[1, 2048], [2048]], "Ev Idx": 1260
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765572.256, "dur": 1.666,
    "args": {
      "External id": 2484462,"Record function id": 0, "Concrete Inputs": ["", "[2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[1, 2048], []], "Ev Idx": 1261
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765580.635, "dur": 4.005,
    "args": {
      "External id": 2484463,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1262
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765583.406, "dur": 1.127,
    "args": {
      "External id": 2484464,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1263
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765603.964, "dur": 16.560,
    "args": {
      "External id": 2484465,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 1264
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765637.037, "dur": 9.068,
    "args": {
      "External id": 2484466,"Record function id": 0, "Ev Idx": 1265
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765639.296, "dur": 6.232,
    "args": {
      "External id": 2484467,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1266
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765641.291, "dur": 3.168,
    "args": {
      "External id": 2484468,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1267
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765642.393, "dur": 1.962,
    "args": {
      "External id": 2484469,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1268
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: AddBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765650.186, "dur": 7.337,
    "args": {
      "External id": 2484470,"Record function id": 0, "Sequence number": 24740511, "Fwd thread id": 1, "Ev Idx": 1269
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "AddBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765652.052, "dur": 1.211,
    "args": {
      "External id": 2484471,"Sequence number": 24740511, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1270
    }
  },
  {
    "ph": "f", "id": 137, "pid": 1336754, "tid": 1381179, "ts": 1514656765652.052,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SwiGLULinearFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765661.335, "dur": 461.243,
    "args": {
      "External id": 2484472,"Record function id": 0, "Sequence number": 24740510, "Fwd thread id": 1, "Ev Idx": 1271
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SwiGLULinearFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765665.027, "dur": 442.907,
    "args": {
      "External id": 2484473,"Sequence number": 24740510, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1272
    }
  },
  {
    "ph": "f", "id": 138, "pid": 1336754, "tid": 1381179, "ts": 1514656765665.027,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765697.929, "dur": 7.547,
    "args": {
      "External id": 2484474,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1273
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_reshape_alias", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765702.336, "dur": 2.842,
    "args": {
      "External id": 2484475,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]", "[8192, 1]"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList"], "Input Strides": [[33554432, 8192, 1], [], []], "Input Dims": [[16, 4096, 2048], [], []], "Ev Idx": 1274
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765708.505, "dur": 6.145,
    "args": {
      "External id": 2484476,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 1275
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765709.788, "dur": 4.230,
    "args": {
      "External id": 2484477,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 1276
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765711.215, "dur": 2.640,
    "args": {
      "External id": 2484478,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 1277
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765718.084, "dur": 87.227,
    "args": {
      "External id": 2484479,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8192, 1], [1, 5632], []], "Input Dims": [[65536, 2048], [5632, 2048], []], "Ev Idx": 1278
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765721.188, "dur": 4.551,
    "args": {
      "External id": 2484480,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 5632]], "Input Dims": [[5632, 2048]], "Ev Idx": 1279
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765722.055, "dur": 3.226,
    "args": {
      "External id": 2484481,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 5632], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 1280
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765724.658, "dur": 0.520,
    "args": {
      "External id": 2484482,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[5632, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 5632], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 1281
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765726.760, "dur": 78.087,
    "args": {
      "External id": 2484483,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8192, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 1282
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765727.897, "dur": 76.158,
    "args": {
      "External id": 2484484,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8192, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 1283
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765808.952, "dur": 2.642,
    "args": {
      "External id": 2484485,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [23068672, 5632, 1]], "Input Dims": [[65536, 5632], [16, 4096, 5632]], "Ev Idx": 1284
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765809.970, "dur": 1.513,
    "args": {
      "External id": 2484486,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 1285
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765842.484, "dur": 6.161,
    "args": {
      "External id": 2484487,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1286
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765851.669, "dur": 2.142,
    "args": {
      "External id": 2484488,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1287
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765854.456, "dur": 1.795,
    "args": {
      "External id": 2484489,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1288
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765890.100, "dur": 2.004,
    "args": {
      "External id": 2484490,"Record function id": 0, "Concrete Inputs": ["", "[-1, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 1289
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765890.802, "dur": 1.149,
    "args": {
      "External id": 2484491,"Record function id": 0, "Concrete Inputs": ["", "[-1, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 1290
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::einsum", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765913.805, "dur": 172.182,
    "args": {
      "External id": 2484492,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["", "TensorList", ""], "Input Strides": [[], [[8192, 1], [5632, 1]], []], "Input Dims": [[], [[65536, 2048], [65536, 5632]], []], "Ev Idx": 1291
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765918.470, "dur": 11.303,
    "args": {
      "External id": 2484493,"Record function id": 0, "Concrete Inputs": ["", "2"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1292
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765923.288, "dur": 5.500,
    "args": {
      "External id": 2484494,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048, 1]", "[8192, 1, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[8192, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 1293
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765931.121, "dur": 6.948,
    "args": {
      "External id": 2484495,"Record function id": 0, "Concrete Inputs": ["", "[1, 2, 0]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8192, 1, 1], []], "Input Dims": [[65536, 2048, 1], []], "Ev Idx": 1294
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765936.381, "dur": 0.630,
    "args": {
      "External id": 2484496,"Record function id": 0, "Concrete Inputs": ["", "[2048, 1, 65536]", "[1, 1, 8192]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[65536, 2048, 1], [], [], []], "Ev Idx": 1295
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765939.199, "dur": 1.667,
    "args": {
      "External id": 2484497,"Record function id": 0, "Concrete Inputs": ["", "2"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 1296
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765939.992, "dur": 0.512,
    "args": {
      "External id": 2484498,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632, 1]", "[5632, 1, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[65536, 5632], [], [], []], "Ev Idx": 1297
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765943.540, "dur": 2.082,
    "args": {
      "External id": 2484499,"Record function id": 0, "Concrete Inputs": ["", "[2, 1, 0]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 1], []], "Input Dims": [[65536, 5632, 1], []], "Ev Idx": 1298
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765944.344, "dur": 0.735,
    "args": {
      "External id": 2484500,"Record function id": 0, "Concrete Inputs": ["", "[1, 5632, 65536]", "[1, 1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1, 1], [], [], []], "Input Dims": [[65536, 5632, 1], [], [], []], "Ev Idx": 1299
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765948.842, "dur": 4.360,
    "args": {
      "External id": 2484501,"Record function id": 0, "Concrete Inputs": ["", "[0, 2, 1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1, 1, 8192], []], "Input Dims": [[2048, 1, 65536], []], "Ev Idx": 1300
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765952.073, "dur": 0.820,
    "args": {
      "External id": 2484502,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536, 1]", "[1, 8192, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1, 8192], [], [], []], "Input Dims": [[2048, 1, 65536], [], [], []], "Ev Idx": 1301
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765953.838, "dur": 6.422,
    "args": {
      "External id": 2484503,"Record function id": 0, "Concrete Inputs": ["", "[1, 2048, 65536]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1, 8192, 1], []], "Input Dims": [[2048, 65536, 1], []], "Ev Idx": 1302
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_reshape_alias", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765958.286, "dur": 1.779,
    "args": {
      "External id": 2484504,"Record function id": 0, "Concrete Inputs": ["", "[1, 2048, 65536]", "[2048, 1, 8192]"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList"], "Input Strides": [[1, 8192, 1], [], []], "Input Dims": [[2048, 65536, 1], [], []], "Ev Idx": 1303
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765963.336, "dur": 4.075,
    "args": {
      "External id": 2484505,"Record function id": 0, "Concrete Inputs": ["", "[2, 1, 0]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1, 1, 5632], []], "Input Dims": [[1, 5632, 65536], []], "Ev Idx": 1304
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765966.625, "dur": 0.530,
    "args": {
      "External id": 2484506,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632, 1]", "[5632, 1, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1, 5632], [], [], []], "Input Dims": [[1, 5632, 65536], [], [], []], "Ev Idx": 1305
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765967.919, "dur": 3.675,
    "args": {
      "External id": 2484507,"Record function id": 0, "Concrete Inputs": ["", "[1, 65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 1], []], "Input Dims": [[65536, 5632, 1], []], "Ev Idx": 1306
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765968.571, "dur": 2.923,
    "args": {
      "External id": 2484508,"Record function id": 0, "Concrete Inputs": ["", "[1, 65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 1], []], "Input Dims": [[65536, 5632, 1], []], "Ev Idx": 1307
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656765972.647, "dur": 97.487,
    "args": {
      "External id": 2484509,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1, 8192], [369098752, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632]], "Ev Idx": 1308
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766074.877, "dur": 1.796,
    "args": {
      "External id": 2484510,"Record function id": 0, "Concrete Inputs": ["", "[2048, 1, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[11534336, 5632, 1], []], "Input Dims": [[1, 2048, 5632], []], "Ev Idx": 1309
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766077.181, "dur": 2.505,
    "args": {
      "External id": 2484511,"Record function id": 0, "Concrete Inputs": ["", "[0, 2, 1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 5632, 1], []], "Input Dims": [[2048, 1, 5632], []], "Ev Idx": 1310
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766078.493, "dur": 0.629,
    "args": {
      "External id": 2484512,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632, 1]", "[5632, 1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 5632, 1], [], [], []], "Input Dims": [[2048, 1, 5632], [], [], []], "Ev Idx": 1311
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766083.961, "dur": 0.852,
    "args": {
      "External id": 2484513,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 5632], []], "Input Dims": [[2048, 5632, 1], []], "Ev Idx": 1312
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766134.079, "dur": 8.686,
    "args": {
      "External id": 2484514,"Record function id": 0, "Ev Idx": 1313
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766136.210, "dur": 5.900,
    "args": {
      "External id": 2484515,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 1314
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766138.237, "dur": 3.028,
    "args": {
      "External id": 2484516,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 1315
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766139.164, "dur": 2.011,
    "args": {
      "External id": 2484517,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 1316
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766146.457, "dur": 7.364,
    "args": {
      "External id": 2484518,"Record function id": 0, "Sequence number": 24740509, "Fwd thread id": 1, "Ev Idx": 1317
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766147.785, "dur": 3.447,
    "args": {
      "External id": 2484519,"Sequence number": 24740509, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[23068672, 5632, 1]], "Input Dims": [[16, 4096, 5632]], "Ev Idx": 1318
    }
  },
  {
    "ph": "f", "id": 139, "pid": 1336754, "tid": 1381179, "ts": 1514656766147.785,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766149.445, "dur": 1.575,
    "args": {
      "External id": 2484520,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 1319
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766149.983, "dur": 0.890,
    "args": {
      "External id": 2484521,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 1320
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766157.372, "dur": 128.696,
    "args": {
      "External id": 2484522,"Record function id": 0, "Sequence number": 24740508, "Fwd thread id": 1, "Ev Idx": 1321
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766158.416, "dur": 119.979,
    "args": {
      "External id": 2484523,"Sequence number": 24740508, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 1322
    }
  },
  {
    "ph": "f", "id": 140, "pid": 1336754, "tid": 1381179, "ts": 1514656766158.416,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766162.052, "dur": 5.684,
    "args": {
      "External id": 2484524,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 1323
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766163.208, "dur": 3.955,
    "args": {
      "External id": 2484525,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[65536, 5632], [], []], "Ev Idx": 1324
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766166.392, "dur": 0.638,
    "args": {
      "External id": 2484526,"Record function id": 0, "Concrete Inputs": ["", "[5632, 65536]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[65536, 5632], [], [], []], "Ev Idx": 1325
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766168.862, "dur": 45.291,
    "args": {
      "External id": 2484527,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048]], "Ev Idx": 1326
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766215.355, "dur": 6.214,
    "args": {
      "External id": 2484528,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1327
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766216.075, "dur": 4.763,
    "args": {
      "External id": 2484529,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 1328
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766219.429, "dur": 1.231,
    "args": {
      "External id": 2484530,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 1329
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766223.082, "dur": 7.902,
    "args": {
      "External id": 2484531,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 1330
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766227.433, "dur": 3.113,
    "args": {
      "External id": 2484532,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 1331
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766228.221, "dur": 2.257,
    "args": {
      "External id": 2484533,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 1332
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766231.626, "dur": 45.993,
    "args": {
      "External id": 2484534,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048]], "Ev Idx": 1333
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766290.847, "dur": 6.253,
    "args": {
      "External id": 2484535,"Record function id": 0, "Sequence number": 24740507, "Fwd thread id": 1, "Ev Idx": 1334
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766292.039, "dur": 3.317,
    "args": {
      "External id": 2484536,"Sequence number": 24740507, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1335
    }
  },
  {
    "ph": "f", "id": 141, "pid": 1336754, "tid": 1381179, "ts": 1514656766292.039,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766293.304, "dur": 1.879,
    "args": {
      "External id": 2484537,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1336
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766294.042, "dur": 0.999,
    "args": {
      "External id": 2484538,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1337
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766300.368, "dur": 11.623,
    "args": {
      "External id": 2484539,"Record function id": 0, "Sequence number": 24740506, "Fwd thread id": 1, "Ev Idx": 1338
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766301.430, "dur": 7.879,
    "args": {
      "External id": 2484540,"Sequence number": 24740506, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 1339
    }
  },
  {
    "ph": "f", "id": 142, "pid": 1336754, "tid": 1381179, "ts": 1514656766301.430,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766304.453, "dur": 4.609,
    "args": {
      "External id": 2484541,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 1340
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766304.977, "dur": 3.592,
    "args": {
      "External id": 2484542,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 1341
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766308.041, "dur": 0.411,
    "args": {
      "External id": 2484543,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 1342
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766315.622, "dur": 4.582,
    "args": {
      "External id": 2484544,"Record function id": 0, "Ev Idx": 1343
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766316.706, "dur": 3.047,
    "args": {
      "External id": 2484545,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1344
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766317.751, "dur": 1.762,
    "args": {
      "External id": 2484546,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1345
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766318.219, "dur": 1.223,
    "args": {
      "External id": 2484547,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1346
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766323.243, "dur": 9.112,
    "args": {
      "External id": 2484548,"Record function id": 0, "Sequence number": 24740505, "Fwd thread id": 1, "Ev Idx": 1347
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766324.339, "dur": 5.054,
    "args": {
      "External id": 2484549,"Sequence number": 24740505, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[23068672, 5632, 1]], "Input Dims": [[16, 4096, 5632]], "Ev Idx": 1348
    }
  },
  {
    "ph": "f", "id": 143, "pid": 1336754, "tid": 1381179, "ts": 1514656766324.339,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766325.800, "dur": 3.450,
    "args": {
      "External id": 2484550,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 1349
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766328.414, "dur": 0.708,
    "args": {
      "External id": 2484551,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 1350
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766335.600, "dur": 113.068,
    "args": {
      "External id": 2484552,"Record function id": 0, "Sequence number": 24740504, "Fwd thread id": 1, "Ev Idx": 1351
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766336.426, "dur": 103.806,
    "args": {
      "External id": 2484553,"Sequence number": 24740504, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 1352
    }
  },
  {
    "ph": "f", "id": 144, "pid": 1336754, "tid": 1381179, "ts": 1514656766336.426,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766340.271, "dur": 4.513,
    "args": {
      "External id": 2484554,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 1353
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766340.659, "dur": 3.720,
    "args": {
      "External id": 2484555,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[65536, 5632], [], []], "Ev Idx": 1354
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766343.614, "dur": 0.675,
    "args": {
      "External id": 2484556,"Record function id": 0, "Concrete Inputs": ["", "[5632, 65536]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[65536, 5632], [], [], []], "Ev Idx": 1355
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766345.552, "dur": 32.807,
    "args": {
      "External id": 2484557,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048]], "Ev Idx": 1356
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766381.497, "dur": 5.101,
    "args": {
      "External id": 2484558,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1357
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766382.105, "dur": 3.995,
    "args": {
      "External id": 2484559,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 1358
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766382.925, "dur": 3.000,
    "args": {
      "External id": 2484560,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 1359
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766387.568, "dur": 3.577,
    "args": {
      "External id": 2484561,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 1360
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766389.020, "dur": 1.459,
    "args": {
      "External id": 2484562,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 1361
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766389.916, "dur": 0.500,
    "args": {
      "External id": 2484563,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 1362
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766393.841, "dur": 45.849,
    "args": {
      "External id": 2484564,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048]], "Ev Idx": 1363
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766454.905, "dur": 35.365,
    "args": {
      "External id": 2484565,"Record function id": 0, "Sequence number": 24740503, "Fwd thread id": 1, "Ev Idx": 1364
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766456.004, "dur": 5.214,
    "args": {
      "External id": 2484566,"Sequence number": 24740503, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1365
    }
  },
  {
    "ph": "f", "id": 145, "pid": 1336754, "tid": 1381179, "ts": 1514656766456.004,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766459.207, "dur": 1.856,
    "args": {
      "External id": 2484567,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1366
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766459.760, "dur": 1.178,
    "args": {
      "External id": 2484568,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1367
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766463.801, "dur": 23.335,
    "args": {
      "External id": 2484569,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 1368
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766494.176, "dur": 10.065,
    "args": {
      "External id": 2484570,"Record function id": 0, "Sequence number": 24740502, "Fwd thread id": 1, "Ev Idx": 1369
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766495.227, "dur": 6.752,
    "args": {
      "External id": 2484571,"Sequence number": 24740502, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 1370
    }
  },
  {
    "ph": "f", "id": 146, "pid": 1336754, "tid": 1381179, "ts": 1514656766495.227,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766496.406, "dur": 5.388,
    "args": {
      "External id": 2484572,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 1371
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766497.254, "dur": 3.983,
    "args": {
      "External id": 2484573,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 1372
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766500.496, "dur": 0.624,
    "args": {
      "External id": 2484574,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 1373
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766508.054, "dur": 7.534,
    "args": {
      "External id": 2484575,"Record function id": 0, "Ev Idx": 1374
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766511.556, "dur": 3.534,
    "args": {
      "External id": 2484576,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1375
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766512.549, "dur": 2.086,
    "args": {
      "External id": 2484577,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1376
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766513.227, "dur": 1.287,
    "args": {
      "External id": 2484578,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1377
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: LayerNormFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766519.778, "dur": 436.582,
    "args": {
      "External id": 2484579,"Record function id": 0, "Sequence number": 24740501, "Fwd thread id": 1, "Ev Idx": 1378
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766525.531, "dur": 395.869,
    "args": {
      "External id": 2484580,"Sequence number": 24740501, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [33554432, 8192, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 1379
    }
  },
  {
    "ph": "f", "id": 147, "pid": 1336754, "tid": 1381179, "ts": 1514656766525.531,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::contiguous", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766563.558, "dur": 38.884,
    "args": {
      "External id": 2484581,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1380
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766564.992, "dur": 37.252,
    "args": {
      "External id": 2484582,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1381
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766568.031, "dur": 6.877,
    "args": {
      "External id": 2484583,"Record function id": 0, "Concrete Inputs": ["", "15", "0", "", "", "0"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[33554432, 8192, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], [], []], "Ev Idx": 1382
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766570.464, "dur": 3.630,
    "args": {
      "External id": 2484584,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 2048]", "15", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1383
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766576.168, "dur": 25.543,
    "args": {
      "External id": 2484585,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 1384
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766616.434, "dur": 6.703,
    "args": {
      "External id": 2484586,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1385
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766619.384, "dur": 3.632,
    "args": {
      "External id": 2484587,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1386
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766626.536, "dur": 1.613,
    "args": {
      "External id": 2484588,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1387
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766627.127, "dur": 0.926,
    "args": {
      "External id": 2484589,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1388
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766642.424, "dur": 2.487,
    "args": {
      "External id": 2484590,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1389
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766655.339, "dur": 3.852,
    "args": {
      "External id": 2484591,"Record function id": 0, "Concrete Inputs": ["[132, 2048]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1390
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766805.238, "dur": 3.472,
    "args": {
      "External id": 2484592,"Record function id": 0, "Concrete Inputs": ["", "[1, -1, 2048]"], "Input type": ["float", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[132, 2048], []], "Ev Idx": 1391
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766812.848, "dur": 36.696,
    "args": {
      "External id": 2484593,"Record function id": 0, "Concrete Inputs": ["", "[1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[270336, 2048, 1], [], [], []], "Input Dims": [[1, 132, 2048], [], [], []], "Ev Idx": 1392
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766822.589, "dur": 0.901,
    "args": {
      "External id": 2484594,"Record function id": 0, "Concrete Inputs": ["", "[1, 1, 2048]", "[2048, 0, 1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[1, 2048], [], [], []], "Ev Idx": 1393
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766854.831, "dur": 30.098,
    "args": {
      "External id": 2484595,"Record function id": 0, "Concrete Inputs": ["", "", "15", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], []], "Ev Idx": 1394
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766856.423, "dur": 28.291,
    "args": {
      "External id": 2484596,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], [], []], "Ev Idx": 1395
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766859.849, "dur": 3.865,
    "args": {
      "External id": 2484597,"Record function id": 0, "Concrete Inputs": ["[1, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1396
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766867.516, "dur": 16.691,
    "args": {
      "External id": 2484598,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "float", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[1, 2048], [1, 2048], []], "Ev Idx": 1397
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766889.256, "dur": 5.078,
    "args": {
      "External id": 2484599,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1]], "Input Dims": [[1, 2048], [2048]], "Ev Idx": 1398
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766893.202, "dur": 0.963,
    "args": {
      "External id": 2484600,"Record function id": 0, "Concrete Inputs": ["", "[2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[1, 2048], []], "Ev Idx": 1399
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766900.269, "dur": 4.393,
    "args": {
      "External id": 2484601,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1400
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766900.996, "dur": 3.532,
    "args": {
      "External id": 2484602,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1401
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766906.631, "dur": 4.028,
    "args": {
      "External id": 2484603,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1402
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766909.400, "dur": 1.172,
    "args": {
      "External id": 2484604,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1403
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766937.318, "dur": 17.489,
    "args": {
      "External id": 2484605,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 1404
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766968.380, "dur": 8.092,
    "args": {
      "External id": 2484606,"Record function id": 0, "Ev Idx": 1405
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766970.427, "dur": 5.269,
    "args": {
      "External id": 2484607,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1406
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766972.213, "dur": 2.492,
    "args": {
      "External id": 2484608,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1407
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766973.106, "dur": 1.522,
    "args": {
      "External id": 2484609,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1408
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766979.780, "dur": 53.276,
    "args": {
      "External id": 2484610,"Record function id": 0, "Sequence number": 24740500, "Fwd thread id": 1, "Ev Idx": 1409
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766980.732, "dur": 46.964,
    "args": {
      "External id": 2484611,"Sequence number": 24740500, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1410
    }
  },
  {
    "ph": "f", "id": 148, "pid": 1336754, "tid": 1381179, "ts": 1514656766980.732,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656766982.557, "dur": 44.947,
    "args": {
      "External id": 2484612,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1411
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767024.674, "dur": 2.467,
    "args": {
      "External id": 2484613,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1412
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767038.253, "dur": 138.037,
    "args": {
      "External id": 2484614,"Record function id": 0, "Sequence number": 24740499, "Fwd thread id": 1, "Ev Idx": 1413
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767039.711, "dur": 129.050,
    "args": {
      "External id": 2484615,"Sequence number": 24740499, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1414
    }
  },
  {
    "ph": "f", "id": 149, "pid": 1336754, "tid": 1381179, "ts": 1514656767039.711,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767043.700, "dur": 5.440,
    "args": {
      "External id": 2484616,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1415
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767045.279, "dur": 3.202,
    "args": {
      "External id": 2484617,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 1416
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767047.196, "dur": 1.108,
    "args": {
      "External id": 2484618,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 1417
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767050.351, "dur": 58.662,
    "args": {
      "External id": 2484619,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 1418
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767112.239, "dur": 7.263,
    "args": {
      "External id": 2484620,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1419
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767112.908, "dur": 6.052,
    "args": {
      "External id": 2484621,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1420
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767116.049, "dur": 2.727,
    "args": {
      "External id": 2484622,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1421
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767121.056, "dur": 3.370,
    "args": {
      "External id": 2484623,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1422
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767122.582, "dur": 1.304,
    "args": {
      "External id": 2484624,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1423
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767123.301, "dur": 0.480,
    "args": {
      "External id": 2484625,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1424
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767127.176, "dur": 40.686,
    "args": {
      "External id": 2484626,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 1425
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767181.279, "dur": 12.044,
    "args": {
      "External id": 2484627,"Record function id": 0, "Sequence number": 24740498, "Fwd thread id": 1, "Ev Idx": 1426
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767182.652, "dur": 7.723,
    "args": {
      "External id": 2484628,"Sequence number": 24740498, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1427
    }
  },
  {
    "ph": "f", "id": 150, "pid": 1336754, "tid": 1381179, "ts": 1514656767182.652,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767183.841, "dur": 6.348,
    "args": {
      "External id": 2484629,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1428
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767186.227, "dur": 3.829,
    "args": {
      "External id": 2484630,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1429
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767196.663, "dur": 9.527,
    "args": {
      "External id": 2484631,"Record function id": 0, "Sequence number": 24740497, "Fwd thread id": 1, "Ev Idx": 1430
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767197.641, "dur": 6.479,
    "args": {
      "External id": 2484632,"Sequence number": 24740497, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1431
    }
  },
  {
    "ph": "f", "id": 151, "pid": 1336754, "tid": 1381179, "ts": 1514656767197.641,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767198.917, "dur": 4.969,
    "args": {
      "External id": 2484633,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1432
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767199.543, "dur": 3.858,
    "args": {
      "External id": 2484634,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1433
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767202.660, "dur": 0.625,
    "args": {
      "External id": 2484635,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1434
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767210.427, "dur": 5.710,
    "args": {
      "External id": 2484636,"Record function id": 0, "Ev Idx": 1435
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767211.765, "dur": 3.876,
    "args": {
      "External id": 2484637,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1436
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767213.013, "dur": 2.338,
    "args": {
      "External id": 2484638,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1437
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767213.918, "dur": 1.319,
    "args": {
      "External id": 2484639,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1438
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767219.000, "dur": 6.540,
    "args": {
      "External id": 2484640,"Record function id": 0, "Sequence number": 24740496, "Fwd thread id": 1, "Ev Idx": 1439
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767219.926, "dur": 3.113,
    "args": {
      "External id": 2484641,"Sequence number": 24740496, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1440
    }
  },
  {
    "ph": "f", "id": 152, "pid": 1336754, "tid": 1381179, "ts": 1514656767219.926,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767221.051, "dur": 1.817,
    "args": {
      "External id": 2484642,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1441
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767221.592, "dur": 1.137,
    "args": {
      "External id": 2484643,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1442
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: FlashAttnFuncBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767229.642, "dur": 296.892,
    "args": {
      "External id": 2484644,"Record function id": 0, "Sequence number": 24740495, "Fwd thread id": 1, "Ev Idx": 1443
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FlashAttnFuncBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767232.921, "dur": 271.602,
    "args": {
      "External id": 2484645,"Sequence number": 24740495, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1444
    }
  },
  {
    "ph": "f", "id": 153, "pid": 1336754, "tid": 1381179, "ts": 1514656767232.921,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767250.722, "dur": 8.532,
    "args": {
      "External id": 2484646,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 1445
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767255.209, "dur": 3.603,
    "args": {
      "External id": 2484647,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1446
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767261.796, "dur": 4.826,
    "args": {
      "External id": 2484648,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 1447
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767264.522, "dur": 1.877,
    "args": {
      "External id": 2484649,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1448
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767268.180, "dur": 3.155,
    "args": {
      "External id": 2484650,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 1449
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767269.039, "dur": 2.038,
    "args": {
      "External id": 2484651,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1450
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767295.500, "dur": 184.588,
    "args": {
      "External id": 2484652,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 1451
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767380.781, "dur": 5.482,
    "args": {
      "External id": 2484653,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1452
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767390.846, "dur": 4.038,
    "args": {
      "External id": 2484654,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1453
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767491.930, "dur": 3.208,
    "args": {
      "External id": 2484655,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1454
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767498.030, "dur": 0.843,
    "args": {
      "External id": 2484656,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1455
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767500.981, "dur": 0.832,
    "args": {
      "External id": 2484657,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1456
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RotaryEmbeddingFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767553.238, "dur": 233.137,
    "args": {
      "External id": 2484658,"Record function id": 0, "Sequence number": 24740494, "Fwd thread id": 1, "Ev Idx": 1457
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767555.337, "dur": 223.444,
    "args": {
      "External id": 2484659,"Sequence number": 24740494, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1458
    }
  },
  {
    "ph": "f", "id": 154, "pid": 1336754, "tid": 1381179, "ts": 1514656767555.337,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767575.912, "dur": 55.227,
    "args": {
      "External id": 2484660,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1459
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767580.614, "dur": 4.122,
    "args": {
      "External id": 2484661,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1460
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767586.285, "dur": 44.104,
    "args": {
      "External id": 2484662,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], []], "Ev Idx": 1461
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767641.320, "dur": 4.380,
    "args": {
      "External id": 2484663,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 1462
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767643.016, "dur": 2.367,
    "args": {
      "External id": 2484664,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1463
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RotaryEmbeddingFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767793.142, "dur": 150.573,
    "args": {
      "External id": 2484665,"Record function id": 0, "Sequence number": 24740493, "Fwd thread id": 1, "Ev Idx": 1464
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767797.140, "dur": 139.524,
    "args": {
      "External id": 2484666,"Sequence number": 24740493, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1465
    }
  },
  {
    "ph": "f", "id": 155, "pid": 1336754, "tid": 1381179, "ts": 1514656767797.140,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767809.573, "dur": 32.014,
    "args": {
      "External id": 2484667,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1466
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767812.027, "dur": 2.560,
    "args": {
      "External id": 2484668,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1467
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767815.567, "dur": 25.448,
    "args": {
      "External id": 2484669,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], []], "Ev Idx": 1468
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767848.039, "dur": 4.738,
    "args": {
      "External id": 2484670,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 1469
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767849.281, "dur": 3.165,
    "args": {
      "External id": 2484671,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1470
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767949.918, "dur": 17.495,
    "args": {
      "External id": 2484672,"Record function id": 0, "Sequence number": 24740492, "Fwd thread id": 1, "Ev Idx": 1471
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767951.528, "dur": 12.887,
    "args": {
      "External id": 2484673,"Sequence number": 24740492, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1472
    }
  },
  {
    "ph": "f", "id": 156, "pid": 1336754, "tid": 1381179, "ts": 1514656767951.528,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767956.592, "dur": 7.587,
    "args": {
      "External id": 2484674,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1473
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767957.894, "dur": 6.073,
    "args": {
      "External id": 2484675,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1474
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767970.671, "dur": 7.342,
    "args": {
      "External id": 2484676,"Record function id": 0, "Sequence number": 24740491, "Fwd thread id": 1, "Ev Idx": 1475
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767971.672, "dur": 3.774,
    "args": {
      "External id": 2484677,"Sequence number": 24740491, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1476
    }
  },
  {
    "ph": "f", "id": 157, "pid": 1336754, "tid": 1381179, "ts": 1514656767971.672,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767973.367, "dur": 1.933,
    "args": {
      "External id": 2484678,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1477
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767974.220, "dur": 0.954,
    "args": {
      "External id": 2484679,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1478
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767981.207, "dur": 51.884,
    "args": {
      "External id": 2484680,"Record function id": 0, "Sequence number": 24740490, "Fwd thread id": 1, "Ev Idx": 1479
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656767982.428, "dur": 47.293,
    "args": {
      "External id": 2484681,"Sequence number": 24740490, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1480
    }
  },
  {
    "ph": "f", "id": 158, "pid": 1336754, "tid": 1381179, "ts": 1514656767982.428,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768026.708, "dur": 2.814,
    "args": {
      "External id": 2484682,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1481
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768027.622, "dur": 1.651,
    "args": {
      "External id": 2484683,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1482
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768038.373, "dur": 7.374,
    "args": {
      "External id": 2484684,"Record function id": 0, "Sequence number": 24740489, "Fwd thread id": 1, "Ev Idx": 1483
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768039.728, "dur": 3.631,
    "args": {
      "External id": 2484685,"Sequence number": 24740489, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1484
    }
  },
  {
    "ph": "f", "id": 159, "pid": 1336754, "tid": 1381179, "ts": 1514656768039.728,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768041.228, "dur": 1.995,
    "args": {
      "External id": 2484686,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1485
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768041.664, "dur": 1.429,
    "args": {
      "External id": 2484687,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1486
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768049.062, "dur": 164.010,
    "args": {
      "External id": 2484688,"Record function id": 0, "Sequence number": 24740488, "Fwd thread id": 1, "Ev Idx": 1487
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768049.946, "dur": 153.726,
    "args": {
      "External id": 2484689,"Sequence number": 24740488, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1488
    }
  },
  {
    "ph": "f", "id": 160, "pid": 1336754, "tid": 1381179, "ts": 1514656768049.946,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768054.587, "dur": 11.297,
    "args": {
      "External id": 2484690,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1489
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768059.089, "dur": 6.097,
    "args": {
      "External id": 2484691,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 1490
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768063.295, "dur": 1.578,
    "args": {
      "External id": 2484692,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 1491
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768067.506, "dur": 71.426,
    "args": {
      "External id": 2484693,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 1492
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768140.351, "dur": 4.456,
    "args": {
      "External id": 2484694,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1493
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768141.230, "dur": 2.734,
    "args": {
      "External id": 2484695,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1494
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768142.425, "dur": 1.345,
    "args": {
      "External id": 2484696,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1495
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768147.072, "dur": 5.702,
    "args": {
      "External id": 2484697,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1496
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768150.805, "dur": 1.521,
    "args": {
      "External id": 2484698,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1497
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768151.572, "dur": 0.613,
    "args": {
      "External id": 2484699,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1498
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768153.618, "dur": 49.305,
    "args": {
      "External id": 2484700,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 1499
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768217.909, "dur": 9.648,
    "args": {
      "External id": 2484701,"Record function id": 0, "Sequence number": 24740487, "Fwd thread id": 1, "Ev Idx": 1500
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768219.075, "dur": 6.408,
    "args": {
      "External id": 2484702,"Sequence number": 24740487, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1501
    }
  },
  {
    "ph": "f", "id": 161, "pid": 1336754, "tid": 1381179, "ts": 1514656768219.075,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768223.153, "dur": 2.168,
    "args": {
      "External id": 2484703,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1502
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768223.791, "dur": 1.387,
    "args": {
      "External id": 2484704,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1503
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768230.939, "dur": 9.205,
    "args": {
      "External id": 2484705,"Record function id": 0, "Sequence number": 24740486, "Fwd thread id": 1, "Ev Idx": 1504
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768232.121, "dur": 5.880,
    "args": {
      "External id": 2484706,"Sequence number": 24740486, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1505
    }
  },
  {
    "ph": "f", "id": 162, "pid": 1336754, "tid": 1381179, "ts": 1514656768232.121,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768235.232, "dur": 2.564,
    "args": {
      "External id": 2484707,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1506
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768235.802, "dur": 1.501,
    "args": {
      "External id": 2484708,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1507
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768236.754, "dur": 0.409,
    "args": {
      "External id": 2484709,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1508
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768246.307, "dur": 9.448,
    "args": {
      "External id": 2484710,"Record function id": 0, "Ev Idx": 1509
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768248.136, "dur": 6.914,
    "args": {
      "External id": 2484711,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1510
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768250.813, "dur": 3.796,
    "args": {
      "External id": 2484712,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1511
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768251.915, "dur": 2.573,
    "args": {
      "External id": 2484713,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1512
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768259.032, "dur": 9.667,
    "args": {
      "External id": 2484714,"Record function id": 0, "Sequence number": 24740485, "Fwd thread id": 1, "Ev Idx": 1513
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768262.215, "dur": 4.514,
    "args": {
      "External id": 2484715,"Sequence number": 24740485, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1514
    }
  },
  {
    "ph": "f", "id": 163, "pid": 1336754, "tid": 1381179, "ts": 1514656768262.215,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768263.425, "dur": 3.150,
    "args": {
      "External id": 2484716,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1515
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768265.609, "dur": 0.836,
    "args": {
      "External id": 2484717,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1516
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768272.003, "dur": 96.591,
    "args": {
      "External id": 2484718,"Record function id": 0, "Sequence number": 24740484, "Fwd thread id": 1, "Ev Idx": 1517
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768273.132, "dur": 88.709,
    "args": {
      "External id": 2484719,"Sequence number": 24740484, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1518
    }
  },
  {
    "ph": "f", "id": 164, "pid": 1336754, "tid": 1381179, "ts": 1514656768273.132,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768276.219, "dur": 2.628,
    "args": {
      "External id": 2484720,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1519
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768277.007, "dur": 1.446,
    "args": {
      "External id": 2484721,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 1520
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768277.842, "dur": 0.484,
    "args": {
      "External id": 2484722,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 1521
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768279.517, "dur": 28.570,
    "args": {
      "External id": 2484723,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 1522
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768310.951, "dur": 3.408,
    "args": {
      "External id": 2484724,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1523
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768311.463, "dur": 2.448,
    "args": {
      "External id": 2484725,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1524
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768312.444, "dur": 1.327,
    "args": {
      "External id": 2484726,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1525
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768315.497, "dur": 7.646,
    "args": {
      "External id": 2484727,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1526
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768316.746, "dur": 5.701,
    "args": {
      "External id": 2484728,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1527
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768319.729, "dur": 2.628,
    "args": {
      "External id": 2484729,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1528
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768325.610, "dur": 35.558,
    "args": {
      "External id": 2484730,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 1529
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768375.022, "dur": 37.635,
    "args": {
      "External id": 2484731,"Record function id": 0, "Sequence number": 24740483, "Fwd thread id": 1, "Ev Idx": 1530
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768376.107, "dur": 8.673,
    "args": {
      "External id": 2484732,"Sequence number": 24740483, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1531
    }
  },
  {
    "ph": "f", "id": 165, "pid": 1336754, "tid": 1381179, "ts": 1514656768376.107,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768377.505, "dur": 7.113,
    "args": {
      "External id": 2484733,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1532
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768383.527, "dur": 0.978,
    "args": {
      "External id": 2484734,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1533
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768387.631, "dur": 22.586,
    "args": {
      "External id": 2484735,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 1534
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768416.398, "dur": 33.599,
    "args": {
      "External id": 2484736,"Record function id": 0, "Sequence number": 24740482, "Fwd thread id": 1, "Ev Idx": 1535
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768439.751, "dur": 7.967,
    "args": {
      "External id": 2484737,"Sequence number": 24740482, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1536
    }
  },
  {
    "ph": "f", "id": 166, "pid": 1336754, "tid": 1381179, "ts": 1514656768439.751,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768442.516, "dur": 4.968,
    "args": {
      "External id": 2484738,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1537
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768443.421, "dur": 3.502,
    "args": {
      "External id": 2484739,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1538
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768446.335, "dur": 0.475,
    "args": {
      "External id": 2484740,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1539
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768453.902, "dur": 4.521,
    "args": {
      "External id": 2484741,"Record function id": 0, "Ev Idx": 1540
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768455.077, "dur": 2.778,
    "args": {
      "External id": 2484742,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1541
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768455.930, "dur": 1.521,
    "args": {
      "External id": 2484743,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1542
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768456.429, "dur": 0.915,
    "args": {
      "External id": 2484744,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1543
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768461.699, "dur": 8.351,
    "args": {
      "External id": 2484745,"Record function id": 0, "Sequence number": 24740481, "Fwd thread id": 1, "Ev Idx": 1544
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768462.420, "dur": 5.229,
    "args": {
      "External id": 2484746,"Sequence number": 24740481, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1545
    }
  },
  {
    "ph": "f", "id": 167, "pid": 1336754, "tid": 1381179, "ts": 1514656768462.420,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768463.618, "dur": 3.808,
    "args": {
      "External id": 2484747,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1546
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768466.495, "dur": 0.840,
    "args": {
      "External id": 2484748,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1547
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768473.081, "dur": 127.955,
    "args": {
      "External id": 2484749,"Record function id": 0, "Sequence number": 24740480, "Fwd thread id": 1, "Ev Idx": 1548
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768473.938, "dur": 117.420,
    "args": {
      "External id": 2484750,"Sequence number": 24740480, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1549
    }
  },
  {
    "ph": "f", "id": 168, "pid": 1336754, "tid": 1381179, "ts": 1514656768473.938,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768476.578, "dur": 4.494,
    "args": {
      "External id": 2484751,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1550
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768476.992, "dur": 3.636,
    "args": {
      "External id": 2484752,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 1551
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768480.112, "dur": 0.399,
    "args": {
      "External id": 2484753,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 1552
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768481.834, "dur": 34.852,
    "args": {
      "External id": 2484754,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 1553
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768519.783, "dur": 2.527,
    "args": {
      "External id": 2484755,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1554
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768520.234, "dur": 1.571,
    "args": {
      "External id": 2484756,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1555
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768520.978, "dur": 0.712,
    "args": {
      "External id": 2484757,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1556
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768523.404, "dur": 2.699,
    "args": {
      "External id": 2484758,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1557
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768524.275, "dur": 1.095,
    "args": {
      "External id": 2484759,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1558
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768524.852, "dur": 0.455,
    "args": {
      "External id": 2484760,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1559
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768545.626, "dur": 44.626,
    "args": {
      "External id": 2484761,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 1560
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768607.718, "dur": 32.304,
    "args": {
      "External id": 2484762,"Record function id": 0, "Sequence number": 24740479, "Fwd thread id": 1, "Ev Idx": 1561
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768608.802, "dur": 5.302,
    "args": {
      "External id": 2484763,"Sequence number": 24740479, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1562
    }
  },
  {
    "ph": "f", "id": 169, "pid": 1336754, "tid": 1381179, "ts": 1514656768608.802,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768612.015, "dur": 1.920,
    "args": {
      "External id": 2484764,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1563
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768612.478, "dur": 1.363,
    "args": {
      "External id": 2484765,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1564
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768616.787, "dur": 21.100,
    "args": {
      "External id": 2484766,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 1565
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768643.673, "dur": 9.522,
    "args": {
      "External id": 2484767,"Record function id": 0, "Sequence number": 24740478, "Fwd thread id": 1, "Ev Idx": 1566
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768644.566, "dur": 6.648,
    "args": {
      "External id": 2484768,"Sequence number": 24740478, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1567
    }
  },
  {
    "ph": "f", "id": 170, "pid": 1336754, "tid": 1381179, "ts": 1514656768644.566,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768645.654, "dur": 5.352,
    "args": {
      "External id": 2484769,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1568
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768646.418, "dur": 4.014,
    "args": {
      "External id": 2484770,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1569
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768649.754, "dur": 0.573,
    "args": {
      "External id": 2484771,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1570
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768657.415, "dur": 5.194,
    "args": {
      "External id": 2484772,"Record function id": 0, "Ev Idx": 1571
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768658.710, "dur": 3.372,
    "args": {
      "External id": 2484773,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1572
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768659.797, "dur": 1.895,
    "args": {
      "External id": 2484774,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1573
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768660.307, "dur": 1.266,
    "args": {
      "External id": 2484775,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1574
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: LayerNormFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768666.818, "dur": 410.901,
    "args": {
      "External id": 2484776,"Record function id": 0, "Sequence number": 24740477, "Fwd thread id": 1, "Ev Idx": 1575
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768669.900, "dur": 360.712,
    "args": {
      "External id": 2484777,"Sequence number": 24740477, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1576
    }
  },
  {
    "ph": "f", "id": 171, "pid": 1336754, "tid": 1381179, "ts": 1514656768669.900,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768704.431, "dur": 1.916,
    "args": {
      "External id": 2484778,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1577
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768705.084, "dur": 1.152,
    "args": {
      "External id": 2484779,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1578
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768721.783, "dur": 5.789,
    "args": {
      "External id": 2484780,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1579
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768736.079, "dur": 2.244,
    "args": {
      "External id": 2484781,"Record function id": 0, "Concrete Inputs": ["[132, 2048]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1580
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768879.096, "dur": 1.918,
    "args": {
      "External id": 2484782,"Record function id": 0, "Concrete Inputs": ["", "[1, -1, 2048]"], "Input type": ["float", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[132, 2048], []], "Ev Idx": 1581
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768884.682, "dur": 37.458,
    "args": {
      "External id": 2484783,"Record function id": 0, "Concrete Inputs": ["", "[1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[270336, 2048, 1], [], [], []], "Input Dims": [[1, 132, 2048], [], [], []], "Ev Idx": 1582
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768897.366, "dur": 0.807,
    "args": {
      "External id": 2484784,"Record function id": 0, "Concrete Inputs": ["", "[1, 1, 2048]", "[2048, 0, 1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[1, 2048], [], [], []], "Ev Idx": 1583
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768930.600, "dur": 31.820,
    "args": {
      "External id": 2484785,"Record function id": 0, "Concrete Inputs": ["", "", "15", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], []], "Ev Idx": 1584
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768932.371, "dur": 29.808,
    "args": {
      "External id": 2484786,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], [], []], "Ev Idx": 1585
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768935.734, "dur": 4.495,
    "args": {
      "External id": 2484787,"Record function id": 0, "Concrete Inputs": ["[1, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1586
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768943.900, "dur": 17.627,
    "args": {
      "External id": 2484788,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "float", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[1, 2048], [1, 2048], []], "Ev Idx": 1587
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768966.850, "dur": 4.619,
    "args": {
      "External id": 2484789,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1]], "Input Dims": [[1, 2048], [2048]], "Ev Idx": 1588
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768970.248, "dur": 1.090,
    "args": {
      "External id": 2484790,"Record function id": 0, "Concrete Inputs": ["", "[2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[1, 2048], []], "Ev Idx": 1589
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768977.203, "dur": 1.808,
    "args": {
      "External id": 2484791,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1590
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656768977.817, "dur": 1.100,
    "args": {
      "External id": 2484792,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1591
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656769051.456, "dur": 19.990,
    "args": {
      "External id": 2484793,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 1592
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656769089.772, "dur": 10.359,
    "args": {
      "External id": 2484794,"Record function id": 0, "Ev Idx": 1593
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656769091.803, "dur": 7.561,
    "args": {
      "External id": 2484795,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1594
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656769093.544, "dur": 4.894,
    "args": {
      "External id": 2484796,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1595
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656769096.234, "dur": 2.104,
    "args": {
      "External id": 2484797,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1596
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656769104.946, "dur": 2862.042,
    "args": {
      "External id": 2484798,"Record function id": 0, "Ev Idx": 1597
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.27)", "pid": 1336754, "tid": 1381179,
    "ts": 1514656769139.135, "dur": 1018.904,
    "args": {
      "External id": 2484799,"Record function id": 0, "Ev Idx": 1598
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.26", "pid": 1336754, "tid": 1381179,
    "ts": 1514656769170.387, "dur": 979.008,
    "args": {
      "External id": 2484800,"Record function id": 0, "Ev Idx": 1599
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.26)", "pid": 1336754, "tid": 1381179,
    "ts": 1514656769184.919, "dur": 948.554,
    "args": {
      "External id": 2484801,"Record function id": 0, "Ev Idx": 1600
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656769281.673, "dur": 7.649,
    "args": {
      "External id": 2484802,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1601
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1381179,
    "ts": 1514656769308.261, "dur": 41.544,
    "args": {
      "External id": 2484803,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 1602
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656769315.468, "dur": 3.761,
    "args": {
      "External id": 2484804,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1603
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656769320.420, "dur": 0.505,
    "args": {
      "External id": 2484805,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1604
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656769326.041, "dur": 0.347,
    "args": {
      "External id": 2484806,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1605
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656769327.182, "dur": 0.445,
    "args": {
      "External id": 2484807,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1606
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656769328.254, "dur": 0.320,
    "args": {
      "External id": 2484808,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1607
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656769333.342, "dur": 0.443,
    "args": {
      "External id": 2484809,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1608
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656769334.304, "dur": 0.263,
    "args": {
      "External id": 2484810,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1609
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656769335.441, "dur": 2.552,
    "args": {
      "External id": 2484811,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1610
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656769340.623, "dur": 2.653,
    "args": {
      "External id": 2484812,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1611
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656769367.974, "dur": 36.382,
    "args": {
      "External id": 2484813,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 1612
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1381179,
    "ts": 1514656769448.757, "dur": 143.879,
    "args": {
      "External id": 2484814,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "1", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 1613
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656769462.632, "dur": 7.485,
    "args": {
      "External id": 2484815,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1614
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1381179,
    "ts": 1514656769477.100, "dur": 9.276,
    "args": {
      "External id": 2484816,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 1615
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1381179,
    "ts": 1514656769480.884, "dur": 5.122,
    "args": {
      "External id": 2484817,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "12846080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 1616
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656769483.920, "dur": 0.740,
    "args": {
      "External id": 2484818,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 1617
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1381179,
    "ts": 1514656769493.963, "dur": 49.918,
    "args": {
      "External id": 2484819,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 1618
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656769497.503, "dur": 0.508,
    "args": {
      "External id": 2484820,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1619
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656769498.905, "dur": 2.034,
    "args": {
      "External id": 2484821,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6423296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1620
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656769501.517, "dur": 0.269,
    "args": {
      "External id": 2484822,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6947584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1621
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656769504.988, "dur": 0.371,
    "args": {
      "External id": 2484823,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7471872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1622
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656769507.700, "dur": 0.458,
    "args": {
      "External id": 2484824,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7996160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1623
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656769508.645, "dur": 2.767,
    "args": {
      "External id": 2484825,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8520448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1624
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656769514.241, "dur": 0.685,
    "args": {
      "External id": 2484826,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "8520704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1625
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656769517.936, "dur": 0.383,
    "args": {
      "External id": 2484827,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "9962496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1626
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656769521.378, "dur": 0.393,
    "args": {
      "External id": 2484828,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11404288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1627
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656769557.556, "dur": 26.788,
    "args": {
      "External id": 2484829,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 1628
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656769660.392, "dur": 323.633,
    "args": {
      "External id": 2484830,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 1629
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514656769698.277, "dur": 280.262,
    "args": {
      "External id": 2484831,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 1630, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514656769709.742, "dur": 262.471,
    "args": {
      "External id": 2484832,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 1631
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514656770043.334, "dur": 4.151,
    "args": {
      "External id": 2484833,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 1632, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656770167.399, "dur": 1775.408,
    "args": {
      "External id": 2484834,"Sequence number": 24740476, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1633
    }
  },
  {
    "ph": "f", "id": 172, "pid": 1336754, "tid": 1381179, "ts": 1514656770167.399,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656770302.642, "dur": 109.904,
    "args": {
      "External id": 2484835,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 1634
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656770461.195, "dur": 44.717,
    "args": {
      "External id": 2484836,"kernel_hash": "cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/fu/cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 1635
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656770543.752, "dur": 59.452,
    "args": {
      "External id": 2484837,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 1636
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656770616.914, "dur": 37.041,
    "args": {
      "External id": 2484838,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 1637
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656770660.728, "dur": 44.801,
    "args": {
      "External id": 2484839,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 1638
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656770714.871, "dur": 26.982,
    "args": {
      "External id": 2484840,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 1639
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656770750.465, "dur": 46.677,
    "args": {
      "External id": 2484841,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 1640
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514656770830.305, "dur": 28.772,
    "args": {
      "External id": 2484842,"kernel_hash": "cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/wm/cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 1641
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656770879.589, "dur": 36.780,
    "args": {
      "External id": 2484843,"kernel_hash": "cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/br/cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 1642
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514656770944.030, "dur": 20.269,
    "args": {
      "External id": 2484844,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 1643
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514656770979.501, "dur": 59.724,
    "args": {
      "External id": 2484845,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 1644
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656771052.971, "dur": 34.390,
    "args": {
      "External id": 2484846,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 1645
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656771090.738, "dur": 36.251,
    "args": {
      "External id": 2484847,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 1646
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656771159.967, "dur": 194.018,
    "args": {
      "External id": 2484848,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 1647
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656771242.067, "dur": 7.990,
    "args": {
      "External id": 2484849,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1648
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656771251.907, "dur": 2.346,
    "args": {
      "External id": 2484850,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1649
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514656771396.905, "dur": 33.667,
    "args": {
      "External id": 2484851,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 1650
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514656771449.033, "dur": 15.605,
    "args": {
      "External id": 2484852,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 1651
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656771472.946, "dur": 37.431,
    "args": {
      "External id": 2484853,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 1652
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656771516.264, "dur": 56.380,
    "args": {
      "External id": 2484854,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 1653
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656771582.770, "dur": 24.071,
    "args": {
      "External id": 2484855,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 1654
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656771613.736, "dur": 30.637,
    "args": {
      "External id": 2484856,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 1655
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656771650.254, "dur": 20.870,
    "args": {
      "External id": 2484857,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 1656
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656771678.450, "dur": 29.402,
    "args": {
      "External id": 2484858,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 1657
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336754, "tid": 1381179,
    "ts": 1514656771732.121, "dur": 25.476,
    "args": {
      "External id": 2484859,"kernel_hash": "c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/6w/c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 1658
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514656771777.011, "dur": 28.371,
    "args": {
      "External id": 2484860,"kernel_hash": "c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/4k/c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 1659
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514656771826.929, "dur": 17.245,
    "args": {
      "External id": 2484861,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 1660
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514656771856.627, "dur": 16.002,
    "args": {
      "External id": 2484862,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 1661
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336754, "tid": 1381179,
    "ts": 1514656771894.088, "dur": 19.340,
    "args": {
      "External id": 2484863,"kernel_hash": "cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/eb/cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 1662
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656772026.495, "dur": 17.903,
    "args": {
      "External id": 2484864,"Record function id": 0, "Ev Idx": 1663
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656772031.013, "dur": 12.083,
    "args": {
      "External id": 2484865,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1664
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656772035.594, "dur": 6.022,
    "args": {
      "External id": 2484866,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1665
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656772037.132, "dur": 4.235,
    "args": {
      "External id": 2484867,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1666
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656772050.171, "dur": 7.462,
    "args": {
      "External id": 2484868,"Record function id": 0, "Ev Idx": 1667
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656772053.731, "dur": 3.432,
    "args": {
      "External id": 2484869,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1668
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656772054.843, "dur": 1.643,
    "args": {
      "External id": 2484870,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1669
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656772055.376, "dur": 0.973,
    "args": {
      "External id": 2484871,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1670
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656772060.987, "dur": 4.974,
    "args": {
      "External id": 2484872,"Record function id": 0, "Ev Idx": 1671
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656772062.502, "dur": 3.030,
    "args": {
      "External id": 2484873,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1672
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656772063.253, "dur": 1.754,
    "args": {
      "External id": 2484874,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1673
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656772064.342, "dur": 0.558,
    "args": {
      "External id": 2484875,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1674
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656772069.202, "dur": 6.353,
    "args": {
      "External id": 2484876,"Record function id": 0, "Ev Idx": 1675
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656772070.277, "dur": 4.879,
    "args": {
      "External id": 2484877,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 1676
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656772070.938, "dur": 3.709,
    "args": {
      "External id": 2484878,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 1677
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656772071.468, "dur": 3.088,
    "args": {
      "External id": 2484879,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 1678
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656772078.657, "dur": 4.264,
    "args": {
      "External id": 2484880,"Record function id": 0, "Ev Idx": 1679
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656772079.993, "dur": 2.499,
    "args": {
      "External id": 2484881,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1680
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656772080.525, "dur": 1.494,
    "args": {
      "External id": 2484882,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1681
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656772081.139, "dur": 0.790,
    "args": {
      "External id": 2484883,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1682
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656772086.063, "dur": 3.503,
    "args": {
      "External id": 2484884,"Record function id": 0, "Ev Idx": 1683
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656772087.132, "dur": 2.044,
    "args": {
      "External id": 2484885,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1684
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656772087.652, "dur": 1.004,
    "args": {
      "External id": 2484886,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1685
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656772088.032, "dur": 0.557,
    "args": {
      "External id": 2484887,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1686
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656772092.718, "dur": 4.236,
    "args": {
      "External id": 2484888,"Record function id": 0, "Ev Idx": 1687
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656772094.003, "dur": 2.540,
    "args": {
      "External id": 2484889,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1688
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656772094.564, "dur": 1.399,
    "args": {
      "External id": 2484890,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1689
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656772095.038, "dur": 0.859,
    "args": {
      "External id": 2484891,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1690
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656772100.082, "dur": 6.270,
    "args": {
      "External id": 2484892,"Record function id": 0, "Ev Idx": 1691
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656772101.769, "dur": 4.197,
    "args": {
      "External id": 2484893,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1692
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656772102.240, "dur": 3.240,
    "args": {
      "External id": 2484894,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1693
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656772104.731, "dur": 0.686,
    "args": {
      "External id": 2484895,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1694
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656772111.503, "dur": 4.186,
    "args": {
      "External id": 2484896,"Record function id": 0, "Ev Idx": 1695
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656772112.946, "dur": 2.359,
    "args": {
      "External id": 2484897,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1696
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656772113.638, "dur": 1.230,
    "args": {
      "External id": 2484898,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1697
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656772114.218, "dur": 0.585,
    "args": {
      "External id": 2484899,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1698
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656772119.912, "dur": 171033.688,
    "args": {
      "External id": 2484900,"Record function id": 0, "Sequence number": 24740475, "Fwd thread id": 1, "Ev Idx": 1699
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656772121.216, "dur": 171022.730,
    "args": {
      "External id": 2484901,"Sequence number": 24740475, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1700
    }
  },
  {
    "ph": "f", "id": 173, "pid": 1336754, "tid": 1381179, "ts": 1514656772121.216,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.27)", "pid": 1336754, "tid": 1381179,
    "ts": 1514656772155.388, "dur": 42.473,
    "args": {
      "External id": 2484902,"Record function id": 0, "Ev Idx": 1701
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.27)", "pid": 1336754, "tid": 1381179,
    "ts": 1514656772205.628, "dur": 95.073,
    "args": {
      "External id": 2484903,"Record function id": 0, "Ev Idx": 1702
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.27)", "pid": 1336754, "tid": 1381179,
    "ts": 1514656772307.793, "dur": 170826.780,
    "args": {
      "External id": 2484904,"Record function id": 0, "Ev Idx": 1703
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656772369.292, "dur": 7.879,
    "args": {
      "External id": 2484905,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1704
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656772391.914, "dur": 4.698,
    "args": {
      "External id": 2484906,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 1705
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514656772417.205, "dur": 169818.105,
    "args": {
      "External id": 2484907,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 1706
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514656772432.147, "dur": 169793.058,
    "args": {
      "External id": 2484908,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 1707
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656772518.484, "dur": 6.533,
    "args": {
      "External id": 2484909,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1708
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514656772553.216, "dur": 169625.063,
    "args": {
      "External id": 2484910,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 1709
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514656772555.903, "dur": 169621.416,
    "args": {
      "External id": 2484911,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 1710
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656772559.212, "dur": 8.738,
    "args": {
      "External id": 2484912,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1711
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656772569.804, "dur": 169603.468,
    "args": {
      "External id": 2484913,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 1712
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656942340.118, "dur": 10.690,
    "args": {
      "External id": 2484914,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 1713
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656942343.282, "dur": 7.162,
    "args": {
      "External id": 2484915,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1714
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656942386.858, "dur": 379.087,
    "args": {
      "External id": 2484916,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 1715
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514656942420.933, "dur": 339.582,
    "args": {
      "External id": 2484917,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 1716, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514656942435.962, "dur": 318.218,
    "args": {
      "External id": 2484918,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 1717
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514656942791.319, "dur": 2.624,
    "args": {
      "External id": 2484919,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 1718, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656942856.255, "dur": 6.911,
    "args": {
      "External id": 2484920,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1719
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656942920.103, "dur": 1.324,
    "args": {
      "External id": 2484921,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1720
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656942938.987, "dur": 1.299,
    "args": {
      "External id": 2484922,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1721
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656942956.338, "dur": 1.099,
    "args": {
      "External id": 2484923,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1722
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656942968.809, "dur": 0.812,
    "args": {
      "External id": 2484924,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1723
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656942979.955, "dur": 0.847,
    "args": {
      "External id": 2484925,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1724
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656943032.575, "dur": 1.750,
    "args": {
      "External id": 2484926,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1725
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656943048.615, "dur": 3.708,
    "args": {
      "External id": 2484927,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1726
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656943063.883, "dur": 0.920,
    "args": {
      "External id": 2484928,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1727
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656943170.182, "dur": 2916.046,
    "args": {
      "External id": 2484929,"Record function id": 0, "Ev Idx": 1728
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.26)", "pid": 1336754, "tid": 1381179,
    "ts": 1514656943189.650, "dur": 1142.504,
    "args": {
      "External id": 2484930,"Record function id": 0, "Ev Idx": 1729
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.26)", "pid": 1336754, "tid": 1381179,
    "ts": 1514656943205.459, "dur": 371.778,
    "args": {
      "External id": 2484931,"Record function id": 0, "Ev Idx": 1730
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656943298.669, "dur": 4.082,
    "args": {
      "External id": 2484932,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 1731
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656943306.136, "dur": 0.995,
    "args": {
      "External id": 2484933,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 1732
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656943308.742, "dur": 0.947,
    "args": {
      "External id": 2484934,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 1733
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656943311.237, "dur": 3.096,
    "args": {
      "External id": 2484935,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 1734
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656943318.037, "dur": 1.069,
    "args": {
      "External id": 2484936,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 1735
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656943320.844, "dur": 1.191,
    "args": {
      "External id": 2484937,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 1736
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656943323.367, "dur": 3.306,
    "args": {
      "External id": 2484938,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 1737
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656943328.228, "dur": 0.832,
    "args": {
      "External id": 2484939,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 1738
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656943334.525, "dur": 0.963,
    "args": {
      "External id": 2484940,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 1739
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656943337.447, "dur": 0.668,
    "args": {
      "External id": 2484941,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 1740
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514656943359.970, "dur": 184.615,
    "args": {
      "External id": 2484942,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 1741
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514656943380.242, "dur": 144.375,
    "args": {
      "External id": 2484943,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 1742
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656943404.172, "dur": 12.606,
    "args": {
      "External id": 2484944,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1743
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514656943421.023, "dur": 73.365,
    "args": {
      "External id": 2484945,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 1744
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514656943425.551, "dur": 68.470,
    "args": {
      "External id": 2484946,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 1745
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656943429.005, "dur": 8.627,
    "args": {
      "External id": 2484947,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1746
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656943439.251, "dur": 54.181,
    "args": {
      "External id": 2484948,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 1747
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.25", "pid": 1336754, "tid": 1381179,
    "ts": 1514656943673.611, "dur": 650.153,
    "args": {
      "External id": 2484949,"Record function id": 0, "Ev Idx": 1748
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.25)", "pid": 1336754, "tid": 1381179,
    "ts": 1514656943691.785, "dur": 618.956,
    "args": {
      "External id": 2484950,"Record function id": 0, "Ev Idx": 1749
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656943755.142, "dur": 5.393,
    "args": {
      "External id": 2484951,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1750
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1381179,
    "ts": 1514656943777.346, "dur": 41.811,
    "args": {
      "External id": 2484952,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 1751
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656943783.836, "dur": 3.188,
    "args": {
      "External id": 2484953,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1752
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656943788.240, "dur": 0.423,
    "args": {
      "External id": 2484954,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1753
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656943790.141, "dur": 0.383,
    "args": {
      "External id": 2484955,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1754
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656943795.527, "dur": 0.570,
    "args": {
      "External id": 2484956,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1755
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656943797.124, "dur": 0.541,
    "args": {
      "External id": 2484957,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1756
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656943798.746, "dur": 0.371,
    "args": {
      "External id": 2484958,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1757
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656943804.176, "dur": 2.308,
    "args": {
      "External id": 2484959,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1758
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656943807.240, "dur": 0.328,
    "args": {
      "External id": 2484960,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1759
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656943810.577, "dur": 2.027,
    "args": {
      "External id": 2484961,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1760
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656943839.738, "dur": 36.761,
    "args": {
      "External id": 2484962,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 1761
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1381179,
    "ts": 1514656943909.554, "dur": 155.286,
    "args": {
      "External id": 2484963,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "1", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 1762
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656943919.742, "dur": 4.958,
    "args": {
      "External id": 2484964,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1763
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1381179,
    "ts": 1514656943930.313, "dur": 11.420,
    "args": {
      "External id": 2484965,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 1764
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1381179,
    "ts": 1514656943934.071, "dur": 7.269,
    "args": {
      "External id": 2484966,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "12846080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 1765
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656943939.326, "dur": 0.754,
    "args": {
      "External id": 2484967,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 1766
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1381179,
    "ts": 1514656943948.583, "dur": 31.167,
    "args": {
      "External id": 2484968,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 1767
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656943950.101, "dur": 0.512,
    "args": {
      "External id": 2484969,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1768
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656943953.570, "dur": 1.769,
    "args": {
      "External id": 2484970,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6423296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1769
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656943956.260, "dur": 0.380,
    "args": {
      "External id": 2484971,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6947584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1770
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656943957.119, "dur": 2.654,
    "args": {
      "External id": 2484972,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7471872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1771
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656943964.289, "dur": 0.360,
    "args": {
      "External id": 2484973,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7996160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1772
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656943965.127, "dur": 0.465,
    "args": {
      "External id": 2484974,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8520448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1773
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656943968.097, "dur": 0.522,
    "args": {
      "External id": 2484975,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "8520704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1774
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656943971.290, "dur": 0.720,
    "args": {
      "External id": 2484976,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "9962496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1775
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656943972.688, "dur": 0.396,
    "args": {
      "External id": 2484977,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11404288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1776
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656944030.707, "dur": 25.125,
    "args": {
      "External id": 2484978,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 1777
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656944110.438, "dur": 132.046,
    "args": {
      "External id": 2484979,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 1778
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514656944143.673, "dur": 95.625,
    "args": {
      "External id": 2484980,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 1779, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514656944155.613, "dur": 79.858,
    "args": {
      "External id": 2484981,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 1780
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514656944259.949, "dur": 1.988,
    "args": {
      "External id": 2484982,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 1781, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656944339.230, "dur": 1722.185,
    "args": {
      "External id": 2484983,"Sequence number": 24740474, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1782
    }
  },
  {
    "ph": "f", "id": 174, "pid": 1336754, "tid": 1381179, "ts": 1514656944339.230,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656944451.372, "dur": 123.629,
    "args": {
      "External id": 2484984,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 1783
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656944623.492, "dur": 42.457,
    "args": {
      "External id": 2484985,"kernel_hash": "cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/fu/cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 1784
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656944683.535, "dur": 54.947,
    "args": {
      "External id": 2484986,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 1785
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656944748.907, "dur": 34.471,
    "args": {
      "External id": 2484987,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 1786
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656944792.164, "dur": 47.291,
    "args": {
      "External id": 2484988,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 1787
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656944846.790, "dur": 28.523,
    "args": {
      "External id": 2484989,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 1788
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656944883.506, "dur": 44.756,
    "args": {
      "External id": 2484990,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 1789
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514656944953.164, "dur": 25.138,
    "args": {
      "External id": 2484991,"kernel_hash": "cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/wm/cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 1790
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656945039.824, "dur": 35.413,
    "args": {
      "External id": 2484992,"kernel_hash": "cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/br/cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 1791
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514656945099.818, "dur": 19.482,
    "args": {
      "External id": 2484993,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 1792
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514656945137.394, "dur": 16.417,
    "args": {
      "External id": 2484994,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 1793
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656945162.277, "dur": 34.720,
    "args": {
      "External id": 2484995,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 1794
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656945200.339, "dur": 34.142,
    "args": {
      "External id": 2484996,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 1795
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656945266.187, "dur": 169.931,
    "args": {
      "External id": 2484997,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 1796
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656945347.750, "dur": 5.255,
    "args": {
      "External id": 2484998,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1797
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656945354.891, "dur": 2.289,
    "args": {
      "External id": 2484999,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1798
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514656945475.121, "dur": 24.866,
    "args": {
      "External id": 2485000,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 1799
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514656945513.057, "dur": 31.567,
    "args": {
      "External id": 2485001,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 1800
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656945555.026, "dur": 56.412,
    "args": {
      "External id": 2485002,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 1801
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656945623.708, "dur": 38.249,
    "args": {
      "External id": 2485003,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 1802
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656945675.003, "dur": 21.369,
    "args": {
      "External id": 2485004,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 1803
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656945700.857, "dur": 30.155,
    "args": {
      "External id": 2485005,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 1804
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656945735.831, "dur": 23.662,
    "args": {
      "External id": 2485006,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 1805
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656945766.752, "dur": 29.379,
    "args": {
      "External id": 2485007,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 1806
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336754, "tid": 1381179,
    "ts": 1514656945822.391, "dur": 23.035,
    "args": {
      "External id": 2485008,"kernel_hash": "c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/6w/c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 1807
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514656945865.992, "dur": 27.435,
    "args": {
      "External id": 2485009,"kernel_hash": "c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/4k/c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 1808
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514656945909.142, "dur": 16.634,
    "args": {
      "External id": 2485010,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 1809
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514656945945.402, "dur": 16.138,
    "args": {
      "External id": 2485011,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 1810
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336754, "tid": 1381179,
    "ts": 1514656945975.749, "dur": 53.310,
    "args": {
      "External id": 2485012,"kernel_hash": "cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/eb/cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 1811
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656946110.122, "dur": 15.365,
    "args": {
      "External id": 2485013,"Record function id": 0, "Ev Idx": 1812
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656946113.280, "dur": 11.243,
    "args": {
      "External id": 2485014,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1813
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656946117.906, "dur": 5.667,
    "args": {
      "External id": 2485015,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1814
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656946119.333, "dur": 4.113,
    "args": {
      "External id": 2485016,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1815
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656946129.489, "dur": 4.815,
    "args": {
      "External id": 2485017,"Record function id": 0, "Ev Idx": 1816
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656946131.075, "dur": 2.780,
    "args": {
      "External id": 2485018,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1817
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656946131.677, "dur": 1.596,
    "args": {
      "External id": 2485019,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1818
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656946132.378, "dur": 0.803,
    "args": {
      "External id": 2485020,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1819
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656946137.580, "dur": 6.788,
    "args": {
      "External id": 2485021,"Record function id": 0, "Ev Idx": 1820
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656946138.587, "dur": 5.381,
    "args": {
      "External id": 2485022,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1821
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656946139.271, "dur": 4.061,
    "args": {
      "External id": 2485023,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1822
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656946139.978, "dur": 3.260,
    "args": {
      "External id": 2485024,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1823
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656946147.534, "dur": 4.442,
    "args": {
      "External id": 2485025,"Record function id": 0, "Ev Idx": 1824
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656946148.718, "dur": 2.868,
    "args": {
      "External id": 2485026,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 1825
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656946149.617, "dur": 1.283,
    "args": {
      "External id": 2485027,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 1826
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656946150.091, "dur": 0.744,
    "args": {
      "External id": 2485028,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 1827
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656946155.254, "dur": 3.319,
    "args": {
      "External id": 2485029,"Record function id": 0, "Ev Idx": 1828
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656946156.313, "dur": 1.864,
    "args": {
      "External id": 2485030,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1829
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656946156.782, "dur": 0.973,
    "args": {
      "External id": 2485031,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1830
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656946157.054, "dur": 0.631,
    "args": {
      "External id": 2485032,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1831
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656946161.745, "dur": 5.303,
    "args": {
      "External id": 2485033,"Record function id": 0, "Ev Idx": 1832
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656946164.017, "dur": 2.605,
    "args": {
      "External id": 2485034,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1833
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656946164.488, "dur": 1.303,
    "args": {
      "External id": 2485035,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1834
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656946165.059, "dur": 0.658,
    "args": {
      "External id": 2485036,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1835
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656946170.347, "dur": 3.879,
    "args": {
      "External id": 2485037,"Record function id": 0, "Ev Idx": 1836
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656946171.326, "dur": 2.482,
    "args": {
      "External id": 2485038,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1837
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656946171.801, "dur": 1.211,
    "args": {
      "External id": 2485039,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1838
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656946172.297, "dur": 0.641,
    "args": {
      "External id": 2485040,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1839
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656946177.340, "dur": 5.349,
    "args": {
      "External id": 2485041,"Record function id": 0, "Ev Idx": 1840
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656946178.188, "dur": 4.109,
    "args": {
      "External id": 2485042,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1841
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656946178.755, "dur": 3.111,
    "args": {
      "External id": 2485043,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1842
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656946181.163, "dur": 0.637,
    "args": {
      "External id": 2485044,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1843
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656946185.810, "dur": 3.463,
    "args": {
      "External id": 2485045,"Record function id": 0, "Ev Idx": 1844
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656946186.748, "dur": 2.105,
    "args": {
      "External id": 2485046,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1845
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656946187.393, "dur": 1.056,
    "args": {
      "External id": 2485047,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1846
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656946187.679, "dur": 0.695,
    "args": {
      "External id": 2485048,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1847
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656946192.807, "dur": 42072.341,
    "args": {
      "External id": 2485049,"Record function id": 0, "Sequence number": 24740473, "Fwd thread id": 1, "Ev Idx": 1848
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656946194.190, "dur": 42063.095,
    "args": {
      "External id": 2485050,"Sequence number": 24740473, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1849
    }
  },
  {
    "ph": "f", "id": 175, "pid": 1336754, "tid": 1381179, "ts": 1514656946194.190,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.26)", "pid": 1336754, "tid": 1381179,
    "ts": 1514656946223.728, "dur": 39.164,
    "args": {
      "External id": 2485051,"Record function id": 0, "Ev Idx": 1850
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.26)", "pid": 1336754, "tid": 1381179,
    "ts": 1514656946270.415, "dur": 66.160,
    "args": {
      "External id": 2485052,"Record function id": 0, "Ev Idx": 1851
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.26)", "pid": 1336754, "tid": 1381179,
    "ts": 1514656946344.261, "dur": 41905.327,
    "args": {
      "External id": 2485053,"Record function id": 0, "Ev Idx": 1852
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656946438.488, "dur": 6.310,
    "args": {
      "External id": 2485054,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1853
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656946453.709, "dur": 6.497,
    "args": {
      "External id": 2485055,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 1854
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514656946477.915, "dur": 40888.114,
    "args": {
      "External id": 2485056,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 1855
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514656946494.766, "dur": 40861.565,
    "args": {
      "External id": 2485057,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 1856
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656946565.776, "dur": 15.073,
    "args": {
      "External id": 2485058,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1857
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514656946589.094, "dur": 40730.490,
    "args": {
      "External id": 2485059,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 1858
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514656946591.648, "dur": 40727.273,
    "args": {
      "External id": 2485060,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 1859
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656946597.303, "dur": 6.433,
    "args": {
      "External id": 2485061,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1860
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656946605.450, "dur": 40709.866,
    "args": {
      "External id": 2485062,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 1861
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656987455.060, "dur": 8.930,
    "args": {
      "External id": 2485063,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 1862
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656987457.709, "dur": 5.773,
    "args": {
      "External id": 2485064,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1863
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656987490.793, "dur": 418.653,
    "args": {
      "External id": 2485065,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 1864
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514656987523.586, "dur": 380.441,
    "args": {
      "External id": 2485066,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 1865, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514656987549.143, "dur": 348.586,
    "args": {
      "External id": 2485067,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 1866
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514656987933.574, "dur": 2.985,
    "args": {
      "External id": 2485068,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 1867, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656988033.899, "dur": 7.181,
    "args": {
      "External id": 2485069,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1868
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656988086.889, "dur": 1.351,
    "args": {
      "External id": 2485070,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1869
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656988105.059, "dur": 3.783,
    "args": {
      "External id": 2485071,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1870
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656988122.446, "dur": 1.114,
    "args": {
      "External id": 2485072,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1871
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656988135.238, "dur": 1.075,
    "args": {
      "External id": 2485073,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1872
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656988147.682, "dur": 1.071,
    "args": {
      "External id": 2485074,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1873
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656988158.982, "dur": 2.949,
    "args": {
      "External id": 2485075,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1874
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656988172.373, "dur": 3.193,
    "args": {
      "External id": 2485076,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1875
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656988188.322, "dur": 0.923,
    "args": {
      "External id": 2485077,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1876
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656988281.009, "dur": 2872.192,
    "args": {
      "External id": 2485078,"Record function id": 0, "Ev Idx": 1877
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.25)", "pid": 1336754, "tid": 1381179,
    "ts": 1514656988299.974, "dur": 1096.507,
    "args": {
      "External id": 2485079,"Record function id": 0, "Ev Idx": 1878
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.25)", "pid": 1336754, "tid": 1381179,
    "ts": 1514656988316.045, "dur": 359.823,
    "args": {
      "External id": 2485080,"Record function id": 0, "Ev Idx": 1879
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656988409.361, "dur": 3.723,
    "args": {
      "External id": 2485081,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 1880
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656988416.422, "dur": 0.968,
    "args": {
      "External id": 2485082,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 1881
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656988419.137, "dur": 3.147,
    "args": {
      "External id": 2485083,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 1882
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656988423.723, "dur": 0.872,
    "args": {
      "External id": 2485084,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 1883
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656988427.928, "dur": 0.741,
    "args": {
      "External id": 2485085,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 1884
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656988429.931, "dur": 1.053,
    "args": {
      "External id": 2485086,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 1885
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656988433.930, "dur": 2.729,
    "args": {
      "External id": 2485087,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 1886
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656988438.084, "dur": 1.076,
    "args": {
      "External id": 2485088,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 1887
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656988443.031, "dur": 0.970,
    "args": {
      "External id": 2485089,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 1888
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656988445.239, "dur": 0.904,
    "args": {
      "External id": 2485090,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 1889
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514656988463.713, "dur": 180.317,
    "args": {
      "External id": 2485091,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 1890
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514656988481.286, "dur": 157.412,
    "args": {
      "External id": 2485092,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 1891
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656988499.749, "dur": 16.877,
    "args": {
      "External id": 2485093,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1892
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514656988520.896, "dur": 87.150,
    "args": {
      "External id": 2485094,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 1893
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514656988525.226, "dur": 82.413,
    "args": {
      "External id": 2485095,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 1894
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656988545.110, "dur": 5.700,
    "args": {
      "External id": 2485096,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1895
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656988552.837, "dur": 54.024,
    "args": {
      "External id": 2485097,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 1896
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.24", "pid": 1336754, "tid": 1381179,
    "ts": 1514656988766.051, "dur": 623.261,
    "args": {
      "External id": 2485098,"Record function id": 0, "Ev Idx": 1897
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.24)", "pid": 1336754, "tid": 1381179,
    "ts": 1514656988783.238, "dur": 593.894,
    "args": {
      "External id": 2485099,"Record function id": 0, "Ev Idx": 1898
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656988843.249, "dur": 5.235,
    "args": {
      "External id": 2485100,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1899
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1381179,
    "ts": 1514656988865.205, "dur": 35.088,
    "args": {
      "External id": 2485101,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 1900
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656988870.843, "dur": 1.470,
    "args": {
      "External id": 2485102,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1901
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656988875.436, "dur": 0.559,
    "args": {
      "External id": 2485103,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1902
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656988876.720, "dur": 0.477,
    "args": {
      "External id": 2485104,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1903
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656988879.861, "dur": 0.825,
    "args": {
      "External id": 2485105,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1904
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656988881.261, "dur": 0.783,
    "args": {
      "External id": 2485106,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1905
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656988884.122, "dur": 2.891,
    "args": {
      "External id": 2485107,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1906
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656988889.563, "dur": 0.243,
    "args": {
      "External id": 2485108,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1907
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656988890.498, "dur": 0.323,
    "args": {
      "External id": 2485109,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1908
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656988892.600, "dur": 0.564,
    "args": {
      "External id": 2485110,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1909
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656988910.540, "dur": 34.031,
    "args": {
      "External id": 2485111,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 1910
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1381179,
    "ts": 1514656988974.761, "dur": 163.729,
    "args": {
      "External id": 2485112,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "1", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 1911
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656989022.454, "dur": 4.810,
    "args": {
      "External id": 2485113,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1912
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1381179,
    "ts": 1514656989033.594, "dur": 13.138,
    "args": {
      "External id": 2485114,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 1913
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1381179,
    "ts": 1514656989037.556, "dur": 8.743,
    "args": {
      "External id": 2485115,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "12846080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 1914
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656989042.289, "dur": 2.151,
    "args": {
      "External id": 2485116,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 1915
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1381179,
    "ts": 1514656989054.217, "dur": 39.284,
    "args": {
      "External id": 2485117,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 1916
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656989055.707, "dur": 0.544,
    "args": {
      "External id": 2485118,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1917
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656989059.208, "dur": 0.702,
    "args": {
      "External id": 2485119,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6423296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1918
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656989072.440, "dur": 2.540,
    "args": {
      "External id": 2485120,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6947584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1919
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656989078.631, "dur": 0.540,
    "args": {
      "External id": 2485121,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7471872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1920
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656989079.634, "dur": 0.458,
    "args": {
      "External id": 2485122,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7996160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1921
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656989080.547, "dur": 1.686,
    "args": {
      "External id": 2485123,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8520448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1922
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656989084.359, "dur": 0.266,
    "args": {
      "External id": 2485124,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "8520704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1923
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656989085.074, "dur": 0.342,
    "args": {
      "External id": 2485125,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "9962496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1924
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656989087.598, "dur": 0.273,
    "args": {
      "External id": 2485126,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11404288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1925
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656989105.610, "dur": 24.852,
    "args": {
      "External id": 2485127,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 1926
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656989184.767, "dur": 125.909,
    "args": {
      "External id": 2485128,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 1927
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514656989216.075, "dur": 91.225,
    "args": {
      "External id": 2485129,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 1928, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514656989225.466, "dur": 77.394,
    "args": {
      "External id": 2485130,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 1929
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514656989326.570, "dur": 2.032,
    "args": {
      "External id": 2485131,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 1930, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656989404.165, "dur": 1724.970,
    "args": {
      "External id": 2485132,"Sequence number": 24740472, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1931
    }
  },
  {
    "ph": "f", "id": 176, "pid": 1336754, "tid": 1381179, "ts": 1514656989404.165,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656989518.497, "dur": 123.320,
    "args": {
      "External id": 2485133,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 1932
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656989686.162, "dur": 41.049,
    "args": {
      "External id": 2485134,"kernel_hash": "cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/fu/cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 1933
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656989744.625, "dur": 50.380,
    "args": {
      "External id": 2485135,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 1934
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656989804.355, "dur": 32.992,
    "args": {
      "External id": 2485136,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 1935
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656989846.087, "dur": 45.679,
    "args": {
      "External id": 2485137,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 1936
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656989900.404, "dur": 27.671,
    "args": {
      "External id": 2485138,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 1937
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656989943.594, "dur": 80.484,
    "args": {
      "External id": 2485139,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 1938
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514656990058.089, "dur": 29.592,
    "args": {
      "External id": 2485140,"kernel_hash": "cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/wm/cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 1939
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514656990110.757, "dur": 32.036,
    "args": {
      "External id": 2485141,"kernel_hash": "cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/br/cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 1940
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514656990167.351, "dur": 18.727,
    "args": {
      "External id": 2485142,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 1941
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514656990203.338, "dur": 17.251,
    "args": {
      "External id": 2485143,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 1942
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656990230.361, "dur": 33.648,
    "args": {
      "External id": 2485144,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 1943
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656990267.358, "dur": 35.359,
    "args": {
      "External id": 2485145,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 1944
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656990332.071, "dur": 171.128,
    "args": {
      "External id": 2485146,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 1945
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656990409.386, "dur": 8.940,
    "args": {
      "External id": 2485147,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1946
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656990420.529, "dur": 2.613,
    "args": {
      "External id": 2485148,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1947
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514656990551.719, "dur": 28.472,
    "args": {
      "External id": 2485149,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 1948
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514656990593.520, "dur": 15.206,
    "args": {
      "External id": 2485150,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 1949
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656990617.766, "dur": 39.728,
    "args": {
      "External id": 2485151,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 1950
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656990663.402, "dur": 50.645,
    "args": {
      "External id": 2485152,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 1951
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656990730.072, "dur": 25.657,
    "args": {
      "External id": 2485153,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 1952
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656990762.875, "dur": 34.284,
    "args": {
      "External id": 2485154,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 1953
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656990802.724, "dur": 20.228,
    "args": {
      "External id": 2485155,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 1954
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514656990829.690, "dur": 30.105,
    "args": {
      "External id": 2485156,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 1955
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336754, "tid": 1381179,
    "ts": 1514656990882.600, "dur": 27.195,
    "args": {
      "External id": 2485157,"kernel_hash": "c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/6w/c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 1956
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514656990929.064, "dur": 25.193,
    "args": {
      "External id": 2485158,"kernel_hash": "c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/4k/c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 1957
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514656990970.023, "dur": 55.548,
    "args": {
      "External id": 2485159,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 1958
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514656991048.357, "dur": 16.964,
    "args": {
      "External id": 2485160,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 1959
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336754, "tid": 1381179,
    "ts": 1514656991083.050, "dur": 18.614,
    "args": {
      "External id": 2485161,"kernel_hash": "cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/eb/cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 1960
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656991175.075, "dur": 15.227,
    "args": {
      "External id": 2485162,"Record function id": 0, "Ev Idx": 1961
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656991178.339, "dur": 11.073,
    "args": {
      "External id": 2485163,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1962
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656991182.291, "dur": 6.170,
    "args": {
      "External id": 2485164,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1963
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656991184.057, "dur": 4.318,
    "args": {
      "External id": 2485165,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1964
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656991194.181, "dur": 6.651,
    "args": {
      "External id": 2485166,"Record function id": 0, "Ev Idx": 1965
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656991197.036, "dur": 3.313,
    "args": {
      "External id": 2485167,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1966
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656991197.849, "dur": 1.883,
    "args": {
      "External id": 2485168,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1967
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656991198.574, "dur": 1.091,
    "args": {
      "External id": 2485169,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1968
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656991204.245, "dur": 6.327,
    "args": {
      "External id": 2485170,"Record function id": 0, "Ev Idx": 1969
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656991205.710, "dur": 4.390,
    "args": {
      "External id": 2485171,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1970
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656991207.966, "dur": 1.692,
    "args": {
      "External id": 2485172,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1971
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656991208.942, "dur": 0.642,
    "args": {
      "External id": 2485173,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1972
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656991213.667, "dur": 6.339,
    "args": {
      "External id": 2485174,"Record function id": 0, "Ev Idx": 1973
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656991214.722, "dur": 4.891,
    "args": {
      "External id": 2485175,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 1974
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656991215.558, "dur": 3.521,
    "args": {
      "External id": 2485176,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 1975
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656991215.844, "dur": 3.165,
    "args": {
      "External id": 2485177,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 1976
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656991223.068, "dur": 3.862,
    "args": {
      "External id": 2485178,"Record function id": 0, "Ev Idx": 1977
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656991224.046, "dur": 2.490,
    "args": {
      "External id": 2485179,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1978
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656991224.535, "dur": 1.376,
    "args": {
      "External id": 2485180,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1979
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656991225.069, "dur": 0.767,
    "args": {
      "External id": 2485181,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1980
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656991230.166, "dur": 3.925,
    "args": {
      "External id": 2485182,"Record function id": 0, "Ev Idx": 1981
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656991231.366, "dur": 2.272,
    "args": {
      "External id": 2485183,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1982
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656991231.825, "dur": 1.199,
    "args": {
      "External id": 2485184,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1983
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656991232.186, "dur": 0.759,
    "args": {
      "External id": 2485185,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1984
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656991237.480, "dur": 7.085,
    "args": {
      "External id": 2485186,"Record function id": 0, "Ev Idx": 1985
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656991238.285, "dur": 5.867,
    "args": {
      "External id": 2485187,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1986
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656991238.986, "dur": 4.735,
    "args": {
      "External id": 2485188,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1987
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656991242.998, "dur": 0.646,
    "args": {
      "External id": 2485189,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1988
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656991247.685, "dur": 3.882,
    "args": {
      "External id": 2485190,"Record function id": 0, "Ev Idx": 1989
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656991248.753, "dur": 2.415,
    "args": {
      "External id": 2485191,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1990
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656991249.236, "dur": 1.369,
    "args": {
      "External id": 2485192,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1991
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656991249.745, "dur": 0.786,
    "args": {
      "External id": 2485193,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1992
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656991254.620, "dur": 3.427,
    "args": {
      "External id": 2485194,"Record function id": 0, "Ev Idx": 1993
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514656991255.646, "dur": 1.999,
    "args": {
      "External id": 2485195,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1994
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656991256.153, "dur": 1.059,
    "args": {
      "External id": 2485196,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1995
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514656991256.426, "dur": 0.717,
    "args": {
      "External id": 2485197,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1996
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656991261.569, "dur": 38523.625,
    "args": {
      "External id": 2485198,"Record function id": 0, "Sequence number": 24740471, "Fwd thread id": 1, "Ev Idx": 1997
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514656991262.830, "dur": 38513.875,
    "args": {
      "External id": 2485199,"Sequence number": 24740471, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1998
    }
  },
  {
    "ph": "f", "id": 177, "pid": 1336754, "tid": 1381179, "ts": 1514656991262.830,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.25)", "pid": 1336754, "tid": 1381179,
    "ts": 1514656991293.882, "dur": 35.882,
    "args": {
      "External id": 2485200,"Record function id": 0, "Ev Idx": 1999
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.25)", "pid": 1336754, "tid": 1381179,
    "ts": 1514656991336.908, "dur": 64.349,
    "args": {
      "External id": 2485201,"Record function id": 0, "Ev Idx": 2000
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.25)", "pid": 1336754, "tid": 1381179,
    "ts": 1514656991411.837, "dur": 38357.137,
    "args": {
      "External id": 2485202,"Record function id": 0, "Ev Idx": 2001
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656991500.523, "dur": 6.697,
    "args": {
      "External id": 2485203,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2002
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514656991516.894, "dur": 4.569,
    "args": {
      "External id": 2485204,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2003
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514656991552.563, "dur": 37355.634,
    "args": {
      "External id": 2485205,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2004
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514656991566.434, "dur": 37332.537,
    "args": {
      "External id": 2485206,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2005
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514656991621.347, "dur": 17.368,
    "args": {
      "External id": 2485207,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2006
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514656991646.194, "dur": 37215.450,
    "args": {
      "External id": 2485208,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 2007
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514656991648.589, "dur": 37212.282,
    "args": {
      "External id": 2485209,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 2008
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514656991651.824, "dur": 9.641,
    "args": {
      "External id": 2485210,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2009
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514656991663.039, "dur": 37194.225,
    "args": {
      "External id": 2485211,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 2010
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657029018.890, "dur": 9.548,
    "args": {
      "External id": 2485212,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 2011
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657029022.105, "dur": 5.768,
    "args": {
      "External id": 2485213,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2012
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657029058.925, "dur": 381.062,
    "args": {
      "External id": 2485214,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 2013
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657029090.700, "dur": 343.736,
    "args": {
      "External id": 2485215,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2014, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514657029104.413, "dur": 323.743,
    "args": {
      "External id": 2485216,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 2015
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657029465.792, "dur": 2.378,
    "args": {
      "External id": 2485217,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2016, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657029544.131, "dur": 6.690,
    "args": {
      "External id": 2485218,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2017
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657029600.007, "dur": 1.432,
    "args": {
      "External id": 2485219,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2018
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657029618.895, "dur": 1.144,
    "args": {
      "External id": 2485220,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2019
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657029634.432, "dur": 3.552,
    "args": {
      "External id": 2485221,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2020
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657029651.361, "dur": 0.942,
    "args": {
      "External id": 2485222,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2021
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657029663.773, "dur": 1.194,
    "args": {
      "External id": 2485223,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2022
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657029676.902, "dur": 0.930,
    "args": {
      "External id": 2485224,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2023
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657029690.367, "dur": 4.790,
    "args": {
      "External id": 2485225,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2024
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657029707.905, "dur": 0.988,
    "args": {
      "External id": 2485226,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2025
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657029801.228, "dur": 2845.576,
    "args": {
      "External id": 2485227,"Record function id": 0, "Ev Idx": 2026
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.24)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657029820.407, "dur": 1096.071,
    "args": {
      "External id": 2485228,"Record function id": 0, "Ev Idx": 2027
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.24)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657029835.756, "dur": 371.977,
    "args": {
      "External id": 2485229,"Record function id": 0, "Ev Idx": 2028
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657029917.350, "dur": 3.875,
    "args": {
      "External id": 2485230,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2029
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657029924.230, "dur": 1.406,
    "args": {
      "External id": 2485231,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2030
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657029927.427, "dur": 0.918,
    "args": {
      "External id": 2485232,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2031
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657029930.712, "dur": 0.819,
    "args": {
      "External id": 2485233,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2032
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657029935.169, "dur": 2.539,
    "args": {
      "External id": 2485234,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2033
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657029939.170, "dur": 1.215,
    "args": {
      "External id": 2485235,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2034
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657029941.980, "dur": 2.637,
    "args": {
      "External id": 2485236,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2035
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657029946.119, "dur": 1.092,
    "args": {
      "External id": 2485237,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2036
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657029950.381, "dur": 1.049,
    "args": {
      "External id": 2485238,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2037
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657029952.965, "dur": 0.929,
    "args": {
      "External id": 2485239,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2038
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657029971.942, "dur": 197.442,
    "args": {
      "External id": 2485240,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2039
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657030023.008, "dur": 141.917,
    "args": {
      "External id": 2485241,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2040
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657030045.704, "dur": 12.943,
    "args": {
      "External id": 2485242,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2041
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514657030062.661, "dur": 71.023,
    "args": {
      "External id": 2485243,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 2042
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657030065.007, "dur": 68.284,
    "args": {
      "External id": 2485244,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 2043
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657030068.811, "dur": 7.575,
    "args": {
      "External id": 2485245,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2044
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657030079.966, "dur": 52.717,
    "args": {
      "External id": 2485246,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 2045
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.23", "pid": 1336754, "tid": 1381179,
    "ts": 1514657030300.292, "dur": 608.934,
    "args": {
      "External id": 2485247,"Record function id": 0, "Ev Idx": 2046
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.23)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657030316.526, "dur": 581.231,
    "args": {
      "External id": 2485248,"Record function id": 0, "Ev Idx": 2047
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657030398.687, "dur": 6.607,
    "args": {
      "External id": 2485249,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2048
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1381179,
    "ts": 1514657030422.303, "dur": 36.445,
    "args": {
      "External id": 2485250,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2049
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657030426.922, "dur": 2.987,
    "args": {
      "External id": 2485251,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2050
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657030431.848, "dur": 0.268,
    "args": {
      "External id": 2485252,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2051
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657030433.637, "dur": 0.245,
    "args": {
      "External id": 2485253,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2052
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657030437.797, "dur": 0.338,
    "args": {
      "External id": 2485254,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2053
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657030439.429, "dur": 0.145,
    "args": {
      "External id": 2485255,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2054
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657030440.741, "dur": 0.148,
    "args": {
      "External id": 2485256,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2055
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657030444.119, "dur": 0.156,
    "args": {
      "External id": 2485257,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2056
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657030445.759, "dur": 2.092,
    "args": {
      "External id": 2485258,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2057
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657030449.382, "dur": 2.025,
    "args": {
      "External id": 2485259,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2058
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657030469.635, "dur": 33.556,
    "args": {
      "External id": 2485260,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2059
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1381179,
    "ts": 1514657030549.972, "dur": 115.493,
    "args": {
      "External id": 2485261,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "1", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 2060
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657030561.380, "dur": 5.026,
    "args": {
      "External id": 2485262,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2061
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1381179,
    "ts": 1514657030571.748, "dur": 10.172,
    "args": {
      "External id": 2485263,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2062
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1381179,
    "ts": 1514657030575.841, "dur": 5.700,
    "args": {
      "External id": 2485264,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "12846080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 2063
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657030579.244, "dur": 0.733,
    "args": {
      "External id": 2485265,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2064
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1381179,
    "ts": 1514657030590.130, "dur": 31.118,
    "args": {
      "External id": 2485266,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2065
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657030592.576, "dur": 0.443,
    "args": {
      "External id": 2485267,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2066
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657030594.737, "dur": 0.308,
    "args": {
      "External id": 2485268,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6423296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2067
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657030596.550, "dur": 1.942,
    "args": {
      "External id": 2485269,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6947584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2068
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657030600.181, "dur": 0.402,
    "args": {
      "External id": 2485270,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7471872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2069
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657030601.904, "dur": 2.244,
    "args": {
      "External id": 2485271,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7996160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2070
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657030607.517, "dur": 0.419,
    "args": {
      "External id": 2485272,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8520448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2071
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657030609.237, "dur": 0.257,
    "args": {
      "External id": 2485273,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "8520704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2072
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657030610.747, "dur": 0.300,
    "args": {
      "External id": 2485274,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "9962496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2073
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657030614.411, "dur": 0.363,
    "args": {
      "External id": 2485275,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11404288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2074
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657030634.330, "dur": 23.187,
    "args": {
      "External id": 2485276,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2075
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657030710.229, "dur": 121.464,
    "args": {
      "External id": 2485277,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 2076
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657030741.283, "dur": 87.007,
    "args": {
      "External id": 2485278,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2077, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514657030750.480, "dur": 73.796,
    "args": {
      "External id": 2485279,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 2078
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657030847.648, "dur": 1.727,
    "args": {
      "External id": 2485280,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2079, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657030924.232, "dur": 1698.255,
    "args": {
      "External id": 2485281,"Sequence number": 24740470, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 2080
    }
  },
  {
    "ph": "f", "id": 178, "pid": 1336754, "tid": 1381179, "ts": 1514657030924.232,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657031072.857, "dur": 107.329,
    "args": {
      "External id": 2485282,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 2081
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514657031224.597, "dur": 40.820,
    "args": {
      "External id": 2485283,"kernel_hash": "cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/fu/cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 2082
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657031281.767, "dur": 50.400,
    "args": {
      "External id": 2485284,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 2083
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657031341.539, "dur": 32.000,
    "args": {
      "External id": 2485285,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2084
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657031381.881, "dur": 45.505,
    "args": {
      "External id": 2485286,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2085
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657031434.189, "dur": 27.846,
    "args": {
      "External id": 2485287,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2086
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657031470.060, "dur": 41.476,
    "args": {
      "External id": 2485288,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2087
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657031551.631, "dur": 26.147,
    "args": {
      "External id": 2485289,"kernel_hash": "cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/wm/cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 2088
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514657031600.634, "dur": 30.321,
    "args": {
      "External id": 2485290,"kernel_hash": "cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/br/cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2089
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657031653.389, "dur": 21.679,
    "args": {
      "External id": 2485291,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 2090
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514657031691.091, "dur": 14.809,
    "args": {
      "External id": 2485292,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 2091
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657031715.570, "dur": 33.325,
    "args": {
      "External id": 2485293,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2092
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657031752.204, "dur": 34.484,
    "args": {
      "External id": 2485294,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2093
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657031821.943, "dur": 208.235,
    "args": {
      "External id": 2485295,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 2094
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657031901.209, "dur": 5.380,
    "args": {
      "External id": 2485296,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2095
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657031908.680, "dur": 2.245,
    "args": {
      "External id": 2485297,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2096
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657032069.556, "dur": 27.903,
    "args": {
      "External id": 2485298,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2097
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657032110.483, "dur": 16.035,
    "args": {
      "External id": 2485299,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2098
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657032135.370, "dur": 45.020,
    "args": {
      "External id": 2485300,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2099
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657032194.111, "dur": 50.896,
    "args": {
      "External id": 2485301,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2100
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657032257.689, "dur": 25.987,
    "args": {
      "External id": 2485302,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2101
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657032290.442, "dur": 29.523,
    "args": {
      "External id": 2485303,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2102
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657032324.947, "dur": 23.283,
    "args": {
      "External id": 2485304,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2103
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657032357.108, "dur": 28.121,
    "args": {
      "External id": 2485305,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2104
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336754, "tid": 1381179,
    "ts": 1514657032409.719, "dur": 21.278,
    "args": {
      "External id": 2485306,"kernel_hash": "c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/6w/c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 2105
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657032452.121, "dur": 22.357,
    "args": {
      "External id": 2485307,"kernel_hash": "c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/4k/c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2106
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657032489.161, "dur": 17.380,
    "args": {
      "External id": 2485308,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 2107
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514657032525.261, "dur": 31.346,
    "args": {
      "External id": 2485309,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 2108
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336754, "tid": 1381179,
    "ts": 1514657032575.063, "dur": 19.896,
    "args": {
      "External id": 2485310,"kernel_hash": "cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/eb/cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 2109
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657032669.432, "dur": 15.388,
    "args": {
      "External id": 2485311,"Record function id": 0, "Ev Idx": 2110
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657032672.992, "dur": 11.000,
    "args": {
      "External id": 2485312,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2111
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657032677.214, "dur": 5.889,
    "args": {
      "External id": 2485313,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2112
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657032678.937, "dur": 4.061,
    "args": {
      "External id": 2485314,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2113
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657032688.865, "dur": 5.195,
    "args": {
      "External id": 2485315,"Record function id": 0, "Ev Idx": 2114
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657032690.461, "dur": 3.183,
    "args": {
      "External id": 2485316,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2115
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657032691.691, "dur": 1.408,
    "args": {
      "External id": 2485317,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2116
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657032692.246, "dur": 0.760,
    "args": {
      "External id": 2485318,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2117
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657032697.469, "dur": 5.094,
    "args": {
      "External id": 2485319,"Record function id": 0, "Ev Idx": 2118
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657032698.939, "dur": 3.212,
    "args": {
      "External id": 2485320,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2119
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657032699.971, "dur": 1.731,
    "args": {
      "External id": 2485321,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2120
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657032700.796, "dur": 0.815,
    "args": {
      "External id": 2485322,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2121
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657032705.785, "dur": 6.861,
    "args": {
      "External id": 2485323,"Record function id": 0, "Ev Idx": 2122
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657032707.457, "dur": 4.758,
    "args": {
      "External id": 2485324,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2123
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657032708.351, "dur": 3.410,
    "args": {
      "External id": 2485325,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2124
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657032708.947, "dur": 2.730,
    "args": {
      "External id": 2485326,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2125
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657032715.743, "dur": 4.669,
    "args": {
      "External id": 2485327,"Record function id": 0, "Ev Idx": 2126
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657032717.534, "dur": 2.479,
    "args": {
      "External id": 2485328,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2127
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657032718.140, "dur": 1.356,
    "args": {
      "External id": 2485329,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2128
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657032718.569, "dur": 0.852,
    "args": {
      "External id": 2485330,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2129
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657032723.497, "dur": 4.213,
    "args": {
      "External id": 2485331,"Record function id": 0, "Ev Idx": 2130
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657032725.201, "dur": 2.111,
    "args": {
      "External id": 2485332,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2131
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657032725.680, "dur": 0.941,
    "args": {
      "External id": 2485333,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2132
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657032726.004, "dur": 0.552,
    "args": {
      "External id": 2485334,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2133
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657032731.190, "dur": 6.207,
    "args": {
      "External id": 2485335,"Record function id": 0, "Ev Idx": 2134
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657032732.450, "dur": 4.526,
    "args": {
      "External id": 2485336,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2135
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657032733.309, "dur": 3.135,
    "args": {
      "External id": 2485337,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2136
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657032735.656, "dur": 0.715,
    "args": {
      "External id": 2485338,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2137
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657032740.552, "dur": 3.576,
    "args": {
      "External id": 2485339,"Record function id": 0, "Ev Idx": 2138
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657032741.747, "dur": 1.976,
    "args": {
      "External id": 2485340,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2139
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657032742.220, "dur": 1.071,
    "args": {
      "External id": 2485341,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2140
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657032742.500, "dur": 0.716,
    "args": {
      "External id": 2485342,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2141
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657032747.159, "dur": 3.710,
    "args": {
      "External id": 2485343,"Record function id": 0, "Ev Idx": 2142
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657032748.519, "dur": 1.939,
    "args": {
      "External id": 2485344,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2143
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657032749.074, "dur": 0.858,
    "args": {
      "External id": 2485345,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2144
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657032749.332, "dur": 0.524,
    "args": {
      "External id": 2485346,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2145
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657032754.721, "dur": 38419.908,
    "args": {
      "External id": 2485347,"Record function id": 0, "Sequence number": 24740469, "Fwd thread id": 1, "Ev Idx": 2146
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657032756.189, "dur": 38409.257,
    "args": {
      "External id": 2485348,"Sequence number": 24740469, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 2147
    }
  },
  {
    "ph": "f", "id": 179, "pid": 1336754, "tid": 1381179, "ts": 1514657032756.189,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.24)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657032785.524, "dur": 39.046,
    "args": {
      "External id": 2485349,"Record function id": 0, "Ev Idx": 2148
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.24)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657032832.480, "dur": 65.640,
    "args": {
      "External id": 2485350,"Record function id": 0, "Ev Idx": 2149
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.24)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657032906.091, "dur": 38251.073,
    "args": {
      "External id": 2485351,"Record function id": 0, "Ev Idx": 2150
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657033030.332, "dur": 8.567,
    "args": {
      "External id": 2485352,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2151
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657033049.924, "dur": 5.142,
    "args": {
      "External id": 2485353,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2152
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514657033070.326, "dur": 37182.488,
    "args": {
      "External id": 2485354,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2153
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514657033084.731, "dur": 37159.185,
    "args": {
      "External id": 2485355,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2154
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657033138.127, "dur": 17.625,
    "args": {
      "External id": 2485356,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2155
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514657033162.621, "dur": 37042.284,
    "args": {
      "External id": 2485357,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 2156
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657033166.997, "dur": 37037.254,
    "args": {
      "External id": 2485358,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 2157
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657033170.994, "dur": 4.814,
    "args": {
      "External id": 2485359,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2158
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657033177.271, "dur": 37023.773,
    "args": {
      "External id": 2485360,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 2159
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657070346.828, "dur": 8.469,
    "args": {
      "External id": 2485361,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 2160
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657070349.796, "dur": 5.123,
    "args": {
      "External id": 2485362,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2161
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657070385.214, "dur": 413.606,
    "args": {
      "External id": 2485363,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 2162
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657070416.127, "dur": 377.337,
    "args": {
      "External id": 2485364,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2163, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514657070427.174, "dur": 360.014,
    "args": {
      "External id": 2485365,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 2164
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657070827.462, "dur": 2.722,
    "args": {
      "External id": 2485366,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2165, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657070892.168, "dur": 6.431,
    "args": {
      "External id": 2485367,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2166
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657070944.083, "dur": 1.524,
    "args": {
      "External id": 2485368,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2167
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657070963.660, "dur": 1.492,
    "args": {
      "External id": 2485369,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2168
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657070979.030, "dur": 35.283,
    "args": {
      "External id": 2485370,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2169
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657071034.586, "dur": 1.274,
    "args": {
      "External id": 2485371,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2170
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657071046.788, "dur": 1.262,
    "args": {
      "External id": 2485372,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2171
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657071059.127, "dur": 1.069,
    "args": {
      "External id": 2485373,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2172
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657071072.455, "dur": 5.019,
    "args": {
      "External id": 2485374,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2173
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657071090.816, "dur": 0.712,
    "args": {
      "External id": 2485375,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2174
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657071192.028, "dur": 2825.776,
    "args": {
      "External id": 2485376,"Record function id": 0, "Ev Idx": 2175
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.23)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657071211.701, "dur": 1071.189,
    "args": {
      "External id": 2485377,"Record function id": 0, "Ev Idx": 2176
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.23)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657071227.047, "dur": 350.505,
    "args": {
      "External id": 2485378,"Record function id": 0, "Ev Idx": 2177
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657071312.939, "dur": 4.147,
    "args": {
      "External id": 2485379,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2178
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657071320.104, "dur": 1.302,
    "args": {
      "External id": 2485380,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2179
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657071323.349, "dur": 0.820,
    "args": {
      "External id": 2485381,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2180
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657071325.869, "dur": 1.135,
    "args": {
      "External id": 2485382,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2181
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657071330.919, "dur": 2.849,
    "args": {
      "External id": 2485383,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2182
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657071335.262, "dur": 1.001,
    "args": {
      "External id": 2485384,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2183
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657071337.840, "dur": 6.402,
    "args": {
      "External id": 2485385,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2184
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657071345.882, "dur": 1.020,
    "args": {
      "External id": 2485386,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2185
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657071350.288, "dur": 1.019,
    "args": {
      "External id": 2485387,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2186
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657071352.816, "dur": 0.889,
    "args": {
      "External id": 2485388,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2187
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657071371.966, "dur": 154.345,
    "args": {
      "External id": 2485389,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2188
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657071387.016, "dur": 134.782,
    "args": {
      "External id": 2485390,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2189
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657071405.376, "dur": 15.719,
    "args": {
      "External id": 2485391,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2190
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514657071424.395, "dur": 69.271,
    "args": {
      "External id": 2485392,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 2191
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657071426.833, "dur": 66.495,
    "args": {
      "External id": 2485393,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 2192
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657071430.915, "dur": 5.397,
    "args": {
      "External id": 2485394,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2193
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657071439.985, "dur": 52.573,
    "args": {
      "External id": 2485395,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 2194
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.22", "pid": 1336754, "tid": 1381179,
    "ts": 1514657071673.437, "dur": 601.746,
    "args": {
      "External id": 2485396,"Record function id": 0, "Ev Idx": 2195
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.22)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657071690.034, "dur": 573.225,
    "args": {
      "External id": 2485397,"Record function id": 0, "Ev Idx": 2196
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657071751.604, "dur": 7.829,
    "args": {
      "External id": 2485398,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2197
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1381179,
    "ts": 1514657071776.287, "dur": 36.115,
    "args": {
      "External id": 2485399,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2198
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657071781.151, "dur": 1.626,
    "args": {
      "External id": 2485400,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2199
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657071786.644, "dur": 0.505,
    "args": {
      "External id": 2485401,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2200
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657071788.655, "dur": 0.388,
    "args": {
      "External id": 2485402,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2201
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657071790.596, "dur": 2.546,
    "args": {
      "External id": 2485403,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2202
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657071795.005, "dur": 0.311,
    "args": {
      "External id": 2485404,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2203
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657071796.597, "dur": 0.254,
    "args": {
      "External id": 2485405,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2204
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657071800.378, "dur": 0.340,
    "args": {
      "External id": 2485406,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2205
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657071802.235, "dur": 2.171,
    "args": {
      "External id": 2485407,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2206
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657071806.239, "dur": 0.273,
    "args": {
      "External id": 2485408,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2207
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657071822.425, "dur": 33.268,
    "args": {
      "External id": 2485409,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2208
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1381179,
    "ts": 1514657071886.980, "dur": 133.048,
    "args": {
      "External id": 2485410,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "1", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 2209
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657071896.559, "dur": 2.885,
    "args": {
      "External id": 2485411,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2210
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1381179,
    "ts": 1514657071904.187, "dur": 11.641,
    "args": {
      "External id": 2485412,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2211
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1381179,
    "ts": 1514657071908.045, "dur": 7.372,
    "args": {
      "External id": 2485413,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "12846080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 2212
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657071911.773, "dur": 2.381,
    "args": {
      "External id": 2485414,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2213
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1381179,
    "ts": 1514657071922.786, "dur": 26.092,
    "args": {
      "External id": 2485415,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2214
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657071924.817, "dur": 0.550,
    "args": {
      "External id": 2485416,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2215
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657071926.986, "dur": 0.380,
    "args": {
      "External id": 2485417,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6423296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2216
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657071928.555, "dur": 0.490,
    "args": {
      "External id": 2485418,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6947584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2217
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657071931.750, "dur": 0.598,
    "args": {
      "External id": 2485419,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7471872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2218
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657071933.793, "dur": 2.090,
    "args": {
      "External id": 2485420,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7996160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2219
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657071937.397, "dur": 1.438,
    "args": {
      "External id": 2485421,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8520448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2220
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657071939.919, "dur": 0.275,
    "args": {
      "External id": 2485422,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "8520704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2221
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657071941.497, "dur": 0.329,
    "args": {
      "External id": 2485423,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "9962496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2222
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657071944.599, "dur": 0.360,
    "args": {
      "External id": 2485424,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11404288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2223
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657071958.370, "dur": 20.618,
    "args": {
      "External id": 2485425,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2224
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657072068.043, "dur": 124.817,
    "args": {
      "External id": 2485426,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 2225
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657072099.079, "dur": 90.391,
    "args": {
      "External id": 2485427,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2226, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514657072108.625, "dur": 76.281,
    "args": {
      "External id": 2485428,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 2227
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657072212.696, "dur": 2.017,
    "args": {
      "External id": 2485429,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2228, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657072290.229, "dur": 1670.719,
    "args": {
      "External id": 2485430,"Sequence number": 24740468, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 2229
    }
  },
  {
    "ph": "f", "id": 180, "pid": 1336754, "tid": 1381179, "ts": 1514657072290.229,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657072399.369, "dur": 103.540,
    "args": {
      "External id": 2485431,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 2230
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514657072564.073, "dur": 41.776,
    "args": {
      "External id": 2485432,"kernel_hash": "cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/fu/cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 2231
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657072623.661, "dur": 55.278,
    "args": {
      "External id": 2485433,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 2232
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657072688.336, "dur": 36.288,
    "args": {
      "External id": 2485434,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2233
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657072733.060, "dur": 49.431,
    "args": {
      "External id": 2485435,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2234
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657072790.618, "dur": 28.545,
    "args": {
      "External id": 2485436,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2235
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657072826.357, "dur": 41.323,
    "args": {
      "External id": 2485437,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2236
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657072893.485, "dur": 23.625,
    "args": {
      "External id": 2485438,"kernel_hash": "cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/wm/cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 2237
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514657072935.286, "dur": 28.496,
    "args": {
      "External id": 2485439,"kernel_hash": "cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/br/cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2238
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657073025.169, "dur": 22.856,
    "args": {
      "External id": 2485440,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 2239
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514657073065.150, "dur": 16.526,
    "args": {
      "External id": 2485441,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 2240
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657073092.150, "dur": 34.552,
    "args": {
      "External id": 2485442,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2241
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657073130.154, "dur": 34.603,
    "args": {
      "External id": 2485443,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2242
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657073193.733, "dur": 164.802,
    "args": {
      "External id": 2485444,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 2243
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657073268.100, "dur": 6.110,
    "args": {
      "External id": 2485445,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2244
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657073275.871, "dur": 2.191,
    "args": {
      "External id": 2485446,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2245
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657073393.703, "dur": 24.520,
    "args": {
      "External id": 2485447,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2246
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657073431.916, "dur": 15.427,
    "args": {
      "External id": 2485448,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2247
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657073454.765, "dur": 34.789,
    "args": {
      "External id": 2485449,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2248
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657073495.014, "dur": 53.153,
    "args": {
      "External id": 2485450,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2249
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657073567.838, "dur": 26.046,
    "args": {
      "External id": 2485451,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2250
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657073600.735, "dur": 46.544,
    "args": {
      "External id": 2485452,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2251
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657073661.206, "dur": 25.261,
    "args": {
      "External id": 2485453,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2252
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657073695.131, "dur": 32.308,
    "args": {
      "External id": 2485454,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2253
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336754, "tid": 1381179,
    "ts": 1514657073754.332, "dur": 24.169,
    "args": {
      "External id": 2485455,"kernel_hash": "c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/6w/c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 2254
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657073799.025, "dur": 28.120,
    "args": {
      "External id": 2485456,"kernel_hash": "c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/4k/c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2255
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657073843.865, "dur": 17.238,
    "args": {
      "External id": 2485457,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 2256
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514657073882.820, "dur": 15.127,
    "args": {
      "External id": 2485458,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 2257
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336754, "tid": 1381179,
    "ts": 1514657073912.968, "dur": 21.060,
    "args": {
      "External id": 2485459,"kernel_hash": "cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/eb/cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 2258
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657074041.360, "dur": 16.027,
    "args": {
      "External id": 2485460,"Record function id": 0, "Ev Idx": 2259
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657074045.132, "dur": 11.209,
    "args": {
      "External id": 2485461,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2260
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657074049.193, "dur": 5.587,
    "args": {
      "External id": 2485462,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2261
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657074050.727, "dur": 3.962,
    "args": {
      "External id": 2485463,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2262
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657074061.302, "dur": 5.339,
    "args": {
      "External id": 2485464,"Record function id": 0, "Ev Idx": 2263
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657074063.051, "dur": 3.118,
    "args": {
      "External id": 2485465,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2264
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657074063.971, "dur": 1.408,
    "args": {
      "External id": 2485466,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2265
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657074064.549, "dur": 0.730,
    "args": {
      "External id": 2485467,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2266
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657074069.968, "dur": 4.806,
    "args": {
      "External id": 2485468,"Record function id": 0, "Ev Idx": 2267
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657074071.772, "dur": 2.587,
    "args": {
      "External id": 2485469,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2268
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657074072.539, "dur": 1.355,
    "args": {
      "External id": 2485470,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2269
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657074073.156, "dur": 0.651,
    "args": {
      "External id": 2485471,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2270
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657074078.070, "dur": 7.024,
    "args": {
      "External id": 2485472,"Record function id": 0, "Ev Idx": 2271
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657074079.344, "dur": 5.350,
    "args": {
      "External id": 2485473,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2272
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657074079.915, "dur": 4.100,
    "args": {
      "External id": 2485474,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2273
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657074080.594, "dur": 3.326,
    "args": {
      "External id": 2485475,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2274
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657074088.196, "dur": 4.080,
    "args": {
      "External id": 2485476,"Record function id": 0, "Ev Idx": 2275
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657074089.427, "dur": 2.448,
    "args": {
      "External id": 2485477,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2276
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657074089.905, "dur": 1.509,
    "args": {
      "External id": 2485478,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2277
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657074090.188, "dur": 1.133,
    "args": {
      "External id": 2485479,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2278
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657074095.361, "dur": 3.695,
    "args": {
      "External id": 2485480,"Record function id": 0, "Ev Idx": 2279
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657074096.633, "dur": 2.023,
    "args": {
      "External id": 2485481,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2280
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657074097.199, "dur": 1.023,
    "args": {
      "External id": 2485482,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2281
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657074097.543, "dur": 0.614,
    "args": {
      "External id": 2485483,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2282
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657074102.374, "dur": 3.734,
    "args": {
      "External id": 2485484,"Record function id": 0, "Ev Idx": 2283
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657074103.659, "dur": 2.054,
    "args": {
      "External id": 2485485,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2284
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657074104.330, "dur": 0.942,
    "args": {
      "External id": 2485486,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2285
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657074104.586, "dur": 0.616,
    "args": {
      "External id": 2485487,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2286
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657074109.143, "dur": 6.539,
    "args": {
      "External id": 2485488,"Record function id": 0, "Ev Idx": 2287
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657074110.772, "dur": 4.511,
    "args": {
      "External id": 2485489,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2288
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657074111.578, "dur": 3.045,
    "args": {
      "External id": 2485490,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2289
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657074114.004, "dur": 0.555,
    "args": {
      "External id": 2485491,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2290
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657074118.781, "dur": 4.327,
    "args": {
      "External id": 2485492,"Record function id": 0, "Ev Idx": 2291
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657074120.285, "dur": 2.401,
    "args": {
      "External id": 2485493,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2292
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657074120.719, "dur": 1.319,
    "args": {
      "External id": 2485494,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2293
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657074121.254, "dur": 0.718,
    "args": {
      "External id": 2485495,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2294
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657074126.552, "dur": 38470.001,
    "args": {
      "External id": 2485496,"Record function id": 0, "Sequence number": 24740467, "Fwd thread id": 1, "Ev Idx": 2295
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657074128.243, "dur": 38458.508,
    "args": {
      "External id": 2485497,"Sequence number": 24740467, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 2296
    }
  },
  {
    "ph": "f", "id": 181, "pid": 1336754, "tid": 1381179, "ts": 1514657074128.243,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.23)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657074159.926, "dur": 38.425,
    "args": {
      "External id": 2485498,"Record function id": 0, "Ev Idx": 2297
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.23)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657074206.580, "dur": 72.321,
    "args": {
      "External id": 2485499,"Record function id": 0, "Ev Idx": 2298
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.23)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657074284.928, "dur": 38292.696,
    "args": {
      "External id": 2485500,"Record function id": 0, "Ev Idx": 2299
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657074375.941, "dur": 9.871,
    "args": {
      "External id": 2485501,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2300
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657074395.230, "dur": 4.457,
    "args": {
      "External id": 2485502,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2301
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514657074416.401, "dur": 37198.149,
    "args": {
      "External id": 2485503,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2302
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514657074429.459, "dur": 37175.512,
    "args": {
      "External id": 2485504,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2303
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657074478.536, "dur": 16.153,
    "args": {
      "External id": 2485505,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2304
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514657074501.163, "dur": 37065.248,
    "args": {
      "External id": 2485506,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 2305
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657074504.600, "dur": 37061.136,
    "args": {
      "External id": 2485507,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 2306
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657074508.594, "dur": 4.686,
    "args": {
      "External id": 2485508,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2307
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657074514.903, "dur": 37047.192,
    "args": {
      "External id": 2485509,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 2308
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657111706.255, "dur": 8.685,
    "args": {
      "External id": 2485510,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 2309
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657111709.381, "dur": 5.275,
    "args": {
      "External id": 2485511,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2310
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657111744.302, "dur": 481.904,
    "args": {
      "External id": 2485512,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 2311
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657111775.818, "dur": 443.650,
    "args": {
      "External id": 2485513,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2312, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514657111788.393, "dur": 345.159,
    "args": {
      "External id": 2485514,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 2313
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657112256.650, "dur": 3.190,
    "args": {
      "External id": 2485515,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2314, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657112329.525, "dur": 10.250,
    "args": {
      "External id": 2485516,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2315
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657112386.535, "dur": 1.102,
    "args": {
      "External id": 2485517,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2316
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657112408.293, "dur": 1.493,
    "args": {
      "External id": 2485518,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2317
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657112424.806, "dur": 1.221,
    "args": {
      "External id": 2485519,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2318
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657112440.312, "dur": 0.886,
    "args": {
      "External id": 2485520,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2319
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657112454.455, "dur": 0.779,
    "args": {
      "External id": 2485521,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2320
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657112469.288, "dur": 0.662,
    "args": {
      "External id": 2485522,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2321
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657112485.684, "dur": 2.117,
    "args": {
      "External id": 2485523,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2322
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657112498.936, "dur": 0.909,
    "args": {
      "External id": 2485524,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2323
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657112613.795, "dur": 2851.050,
    "args": {
      "External id": 2485525,"Record function id": 0, "Ev Idx": 2324
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.22)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657112633.096, "dur": 1083.847,
    "args": {
      "External id": 2485526,"Record function id": 0, "Ev Idx": 2325
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.22)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657112648.989, "dur": 368.745,
    "args": {
      "External id": 2485527,"Record function id": 0, "Ev Idx": 2326
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657112734.518, "dur": 4.307,
    "args": {
      "External id": 2485528,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2327
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657112741.984, "dur": 1.274,
    "args": {
      "External id": 2485529,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2328
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657112746.880, "dur": 1.245,
    "args": {
      "External id": 2485530,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2329
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657112749.816, "dur": 3.064,
    "args": {
      "External id": 2485531,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2330
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657112754.231, "dur": 0.781,
    "args": {
      "External id": 2485532,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2331
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657112756.776, "dur": 0.974,
    "args": {
      "External id": 2485533,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2332
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657112761.060, "dur": 2.007,
    "args": {
      "External id": 2485534,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2333
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657112764.473, "dur": 0.897,
    "args": {
      "External id": 2485535,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2334
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657112766.958, "dur": 0.804,
    "args": {
      "External id": 2485536,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2335
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657112769.316, "dur": 0.924,
    "args": {
      "External id": 2485537,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2336
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657112790.663, "dur": 160.290,
    "args": {
      "External id": 2485538,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2337
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657112806.704, "dur": 139.497,
    "args": {
      "External id": 2485539,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2338
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657112829.455, "dur": 12.870,
    "args": {
      "External id": 2485540,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2339
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514657112845.606, "dur": 70.277,
    "args": {
      "External id": 2485541,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 2340
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657112847.966, "dur": 67.540,
    "args": {
      "External id": 2485542,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 2341
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657112852.207, "dur": 7.829,
    "args": {
      "External id": 2485543,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2342
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657112861.680, "dur": 53.227,
    "args": {
      "External id": 2485544,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 2343
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.21", "pid": 1336754, "tid": 1381179,
    "ts": 1514657113116.446, "dur": 592.359,
    "args": {
      "External id": 2485545,"Record function id": 0, "Ev Idx": 2344
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.21)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657113135.013, "dur": 561.492,
    "args": {
      "External id": 2485546,"Record function id": 0, "Ev Idx": 2345
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657113199.414, "dur": 5.771,
    "args": {
      "External id": 2485547,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2346
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1381179,
    "ts": 1514657113221.299, "dur": 33.020,
    "args": {
      "External id": 2485548,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2347
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657113226.150, "dur": 2.465,
    "args": {
      "External id": 2485549,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2348
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657113230.904, "dur": 0.636,
    "args": {
      "External id": 2485550,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2349
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657113233.598, "dur": 0.552,
    "args": {
      "External id": 2485551,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2350
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657113236.368, "dur": 0.552,
    "args": {
      "External id": 2485552,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2351
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657113238.492, "dur": 0.398,
    "args": {
      "External id": 2485553,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2352
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657113240.367, "dur": 0.379,
    "args": {
      "External id": 2485554,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2353
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657113243.226, "dur": 2.739,
    "args": {
      "External id": 2485555,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2354
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657113247.179, "dur": 0.535,
    "args": {
      "External id": 2485556,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2355
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657113248.708, "dur": 0.575,
    "args": {
      "External id": 2485557,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2356
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657113268.789, "dur": 38.764,
    "args": {
      "External id": 2485558,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2357
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1381179,
    "ts": 1514657113340.790, "dur": 100.641,
    "args": {
      "External id": 2485559,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "1", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 2358
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657113350.402, "dur": 3.415,
    "args": {
      "External id": 2485560,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2359
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1381179,
    "ts": 1514657113358.939, "dur": 9.937,
    "args": {
      "External id": 2485561,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2360
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1381179,
    "ts": 1514657113362.713, "dur": 5.746,
    "args": {
      "External id": 2485562,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "12846080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 2361
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657113366.454, "dur": 0.646,
    "args": {
      "External id": 2485563,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2362
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1381179,
    "ts": 1514657113376.010, "dur": 27.104,
    "args": {
      "External id": 2485564,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2363
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657113377.989, "dur": 0.364,
    "args": {
      "External id": 2485565,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2364
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657113381.064, "dur": 0.526,
    "args": {
      "External id": 2485566,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6423296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2365
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657113383.177, "dur": 0.459,
    "args": {
      "External id": 2485567,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6947584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2366
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657113385.419, "dur": 2.240,
    "args": {
      "External id": 2485568,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7471872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2367
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657113389.383, "dur": 0.721,
    "args": {
      "External id": 2485569,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7996160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2368
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657113391.922, "dur": 0.630,
    "args": {
      "External id": 2485570,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8520448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2369
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657113394.521, "dur": 0.562,
    "args": {
      "External id": 2485571,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "8520704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2370
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657113396.792, "dur": 0.414,
    "args": {
      "External id": 2485572,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "9962496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2371
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657113398.562, "dur": 0.409,
    "args": {
      "External id": 2485573,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11404288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2372
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657113414.240, "dur": 20.283,
    "args": {
      "External id": 2485574,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2373
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657113486.018, "dur": 138.333,
    "args": {
      "External id": 2485575,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 2374
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657113513.017, "dur": 107.446,
    "args": {
      "External id": 2485576,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2375, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514657113523.411, "dur": 92.817,
    "args": {
      "External id": 2485577,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 2376
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657113641.363, "dur": 1.893,
    "args": {
      "External id": 2485578,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2377, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657113724.494, "dur": 1717.116,
    "args": {
      "External id": 2485579,"Sequence number": 24740466, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 2378
    }
  },
  {
    "ph": "f", "id": 182, "pid": 1336754, "tid": 1381179, "ts": 1514657113724.494,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657113832.533, "dur": 103.451,
    "args": {
      "External id": 2485580,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 2379
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514657114019.510, "dur": 45.775,
    "args": {
      "External id": 2485581,"kernel_hash": "cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/fu/cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 2380
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657114085.954, "dur": 57.982,
    "args": {
      "External id": 2485582,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 2381
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657114153.609, "dur": 33.042,
    "args": {
      "External id": 2485583,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2382
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657114195.258, "dur": 45.833,
    "args": {
      "External id": 2485584,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2383
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657114248.741, "dur": 29.009,
    "args": {
      "External id": 2485585,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2384
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657114285.688, "dur": 44.521,
    "args": {
      "External id": 2485586,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2385
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657114353.483, "dur": 22.817,
    "args": {
      "External id": 2485587,"kernel_hash": "cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/wm/cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 2386
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514657114404.377, "dur": 31.596,
    "args": {
      "External id": 2485588,"kernel_hash": "cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/br/cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2387
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657114457.440, "dur": 18.953,
    "args": {
      "External id": 2485589,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 2388
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514657114490.058, "dur": 16.596,
    "args": {
      "External id": 2485590,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 2389
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657114516.658, "dur": 49.202,
    "args": {
      "External id": 2485591,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2390
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657114571.355, "dur": 42.157,
    "args": {
      "External id": 2485592,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2391
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657114647.874, "dur": 176.868,
    "args": {
      "External id": 2485593,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 2392
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657114729.419, "dur": 5.780,
    "args": {
      "External id": 2485594,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2393
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657114737.265, "dur": 2.282,
    "args": {
      "External id": 2485595,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2394
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657114857.505, "dur": 26.844,
    "args": {
      "External id": 2485596,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2395
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657114895.785, "dur": 16.646,
    "args": {
      "External id": 2485597,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2396
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657114930.239, "dur": 42.465,
    "args": {
      "External id": 2485598,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2397
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115018.872, "dur": 48.002,
    "args": {
      "External id": 2485599,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2398
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115079.861, "dur": 25.712,
    "args": {
      "External id": 2485600,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2399
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115111.432, "dur": 31.875,
    "args": {
      "External id": 2485601,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2400
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115150.492, "dur": 21.077,
    "args": {
      "External id": 2485602,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2401
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115179.982, "dur": 31.230,
    "args": {
      "External id": 2485603,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2402
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115230.888, "dur": 24.679,
    "args": {
      "External id": 2485604,"kernel_hash": "c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/6w/c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 2403
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115274.458, "dur": 25.482,
    "args": {
      "External id": 2485605,"kernel_hash": "c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/4k/c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2404
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115315.919, "dur": 22.087,
    "args": {
      "External id": 2485606,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 2405
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115354.639, "dur": 15.647,
    "args": {
      "External id": 2485607,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 2406
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115394.795, "dur": 18.717,
    "args": {
      "External id": 2485608,"kernel_hash": "cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/eb/cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 2407
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115487.641, "dur": 15.731,
    "args": {
      "External id": 2485609,"Record function id": 0, "Ev Idx": 2408
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115491.022, "dur": 11.365,
    "args": {
      "External id": 2485610,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2409
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115495.115, "dur": 6.326,
    "args": {
      "External id": 2485611,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2410
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115497.134, "dur": 4.186,
    "args": {
      "External id": 2485612,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2411
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115507.364, "dur": 5.314,
    "args": {
      "External id": 2485613,"Record function id": 0, "Ev Idx": 2412
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115508.827, "dur": 3.374,
    "args": {
      "External id": 2485614,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2413
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115509.681, "dur": 2.026,
    "args": {
      "External id": 2485615,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2414
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115510.609, "dur": 1.032,
    "args": {
      "External id": 2485616,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2415
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115516.204, "dur": 7.465,
    "args": {
      "External id": 2485617,"Record function id": 0, "Ev Idx": 2416
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115517.628, "dur": 5.634,
    "args": {
      "External id": 2485618,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2417
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115518.370, "dur": 4.441,
    "args": {
      "External id": 2485619,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2418
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115518.971, "dur": 3.756,
    "args": {
      "External id": 2485620,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2419
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115526.867, "dur": 22.007,
    "args": {
      "External id": 2485621,"Record function id": 0, "Ev Idx": 2420
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115543.365, "dur": 4.566,
    "args": {
      "External id": 2485622,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2421
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115544.460, "dur": 2.593,
    "args": {
      "External id": 2485623,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2422
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115545.159, "dur": 1.627,
    "args": {
      "External id": 2485624,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2423
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115553.443, "dur": 5.286,
    "args": {
      "External id": 2485625,"Record function id": 0, "Ev Idx": 2424
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115555.504, "dur": 2.836,
    "args": {
      "External id": 2485626,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2425
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115556.193, "dur": 1.451,
    "args": {
      "External id": 2485627,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2426
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115556.791, "dur": 0.787,
    "args": {
      "External id": 2485628,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2427
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115562.099, "dur": 4.222,
    "args": {
      "External id": 2485629,"Record function id": 0, "Ev Idx": 2428
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115563.478, "dur": 2.430,
    "args": {
      "External id": 2485630,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2429
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115563.999, "dur": 1.486,
    "args": {
      "External id": 2485631,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2430
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115564.429, "dur": 0.943,
    "args": {
      "External id": 2485632,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2431
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115569.724, "dur": 4.217,
    "args": {
      "External id": 2485633,"Record function id": 0, "Ev Idx": 2432
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115571.139, "dur": 2.376,
    "args": {
      "External id": 2485634,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2433
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115571.876, "dur": 1.183,
    "args": {
      "External id": 2485635,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2434
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115572.252, "dur": 0.717,
    "args": {
      "External id": 2485636,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2435
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115577.070, "dur": 3.604,
    "args": {
      "External id": 2485637,"Record function id": 0, "Ev Idx": 2436
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115578.286, "dur": 1.994,
    "args": {
      "External id": 2485638,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2437
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115578.868, "dur": 0.969,
    "args": {
      "External id": 2485639,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2438
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115579.125, "dur": 0.645,
    "args": {
      "External id": 2485640,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2439
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115583.691, "dur": 4.358,
    "args": {
      "External id": 2485641,"Record function id": 0, "Ev Idx": 2440
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115584.964, "dur": 2.688,
    "args": {
      "External id": 2485642,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2441
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115585.482, "dur": 1.556,
    "args": {
      "External id": 2485643,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2442
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115586.285, "dur": 0.663,
    "args": {
      "External id": 2485644,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2443
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115591.872, "dur": 37458.204,
    "args": {
      "External id": 2485645,"Record function id": 0, "Sequence number": 24740465, "Fwd thread id": 1, "Ev Idx": 2444
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115593.334, "dur": 37446.490,
    "args": {
      "External id": 2485646,"Sequence number": 24740465, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 2445
    }
  },
  {
    "ph": "f", "id": 183, "pid": 1336754, "tid": 1381179, "ts": 1514657115593.334,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.22)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115625.560, "dur": 38.914,
    "args": {
      "External id": 2485647,"Record function id": 0, "Ev Idx": 2446
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.22)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115672.588, "dur": 68.643,
    "args": {
      "External id": 2485648,"Record function id": 0, "Ev Idx": 2447
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.22)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115746.950, "dur": 37283.388,
    "args": {
      "External id": 2485649,"Record function id": 0, "Ev Idx": 2448
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115833.879, "dur": 6.976,
    "args": {
      "External id": 2485650,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2449
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115850.565, "dur": 6.486,
    "args": {
      "External id": 2485651,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2450
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115871.607, "dur": 36300.018,
    "args": {
      "External id": 2485652,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2451
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115884.917, "dur": 36277.097,
    "args": {
      "External id": 2485653,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2452
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115934.212, "dur": 15.167,
    "args": {
      "External id": 2485654,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2453
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115956.120, "dur": 36165.942,
    "args": {
      "External id": 2485655,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 2454
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115958.897, "dur": 36162.132,
    "args": {
      "External id": 2485656,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 2455
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115962.818, "dur": 4.971,
    "args": {
      "External id": 2485657,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2456
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657115969.730, "dur": 36147.987,
    "args": {
      "External id": 2485658,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 2457
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657152266.804, "dur": 9.757,
    "args": {
      "External id": 2485659,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 2458
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657152270.421, "dur": 5.766,
    "args": {
      "External id": 2485660,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2459
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657152305.930, "dur": 362.170,
    "args": {
      "External id": 2485661,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 2460
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657152331.875, "dur": 330.503,
    "args": {
      "External id": 2485662,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2461, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514657152343.381, "dur": 312.980,
    "args": {
      "External id": 2485663,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 2462
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657152689.424, "dur": 2.322,
    "args": {
      "External id": 2485664,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2463, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657152751.798, "dur": 6.671,
    "args": {
      "External id": 2485665,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2464
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657152805.611, "dur": 1.464,
    "args": {
      "External id": 2485666,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2465
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657152824.766, "dur": 2.958,
    "args": {
      "External id": 2485667,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2466
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657152841.342, "dur": 0.935,
    "args": {
      "External id": 2485668,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2467
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657152855.675, "dur": 1.056,
    "args": {
      "External id": 2485669,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2468
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657152870.280, "dur": 1.217,
    "args": {
      "External id": 2485670,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2469
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657152884.034, "dur": 2.862,
    "args": {
      "External id": 2485671,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2470
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657152903.587, "dur": 2.065,
    "args": {
      "External id": 2485672,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2471
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657152917.484, "dur": 1.182,
    "args": {
      "External id": 2485673,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2472
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657153066.833, "dur": 2872.022,
    "args": {
      "External id": 2485674,"Record function id": 0, "Ev Idx": 2473
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.21)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657153091.835, "dur": 1097.048,
    "args": {
      "External id": 2485675,"Record function id": 0, "Ev Idx": 2474
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.21)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657153108.751, "dur": 355.899,
    "args": {
      "External id": 2485676,"Record function id": 0, "Ev Idx": 2475
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657153198.481, "dur": 7.727,
    "args": {
      "External id": 2485677,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2476
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657153210.035, "dur": 1.196,
    "args": {
      "External id": 2485678,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2477
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657153213.120, "dur": 3.273,
    "args": {
      "External id": 2485679,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2478
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657153218.130, "dur": 1.235,
    "args": {
      "External id": 2485680,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2479
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657153220.919, "dur": 0.978,
    "args": {
      "External id": 2485681,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2480
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657153223.505, "dur": 1.339,
    "args": {
      "External id": 2485682,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2481
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657153226.432, "dur": 1.669,
    "args": {
      "External id": 2485683,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2482
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657153229.932, "dur": 1.502,
    "args": {
      "External id": 2485684,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2483
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657153232.990, "dur": 1.251,
    "args": {
      "External id": 2485685,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2484
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657153235.710, "dur": 1.162,
    "args": {
      "External id": 2485686,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2485
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657153255.548, "dur": 175.383,
    "args": {
      "External id": 2485687,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2486
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657153290.596, "dur": 135.549,
    "args": {
      "External id": 2485688,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2487
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657153308.041, "dur": 15.012,
    "args": {
      "External id": 2485689,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2488
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514657153326.713, "dur": 69.676,
    "args": {
      "External id": 2485690,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 2489
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657153329.434, "dur": 66.527,
    "args": {
      "External id": 2485691,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 2490
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657153333.527, "dur": 6.758,
    "args": {
      "External id": 2485692,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2491
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657153342.121, "dur": 53.096,
    "args": {
      "External id": 2485693,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 2492
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.20", "pid": 1336754, "tid": 1381179,
    "ts": 1514657153577.212, "dur": 603.594,
    "args": {
      "External id": 2485694,"Record function id": 0, "Ev Idx": 2493
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.20)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657153596.447, "dur": 570.835,
    "args": {
      "External id": 2485695,"Record function id": 0, "Ev Idx": 2494
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657153661.065, "dur": 5.844,
    "args": {
      "External id": 2485696,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2495
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1381179,
    "ts": 1514657153683.246, "dur": 33.448,
    "args": {
      "External id": 2485697,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2496
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657153688.655, "dur": 1.655,
    "args": {
      "External id": 2485698,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2497
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657153692.614, "dur": 0.591,
    "args": {
      "External id": 2485699,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2498
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657153695.605, "dur": 0.775,
    "args": {
      "External id": 2485700,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2499
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657153698.152, "dur": 0.545,
    "args": {
      "External id": 2485701,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2500
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657153700.471, "dur": 0.607,
    "args": {
      "External id": 2485702,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2501
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657153702.815, "dur": 2.707,
    "args": {
      "External id": 2485703,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2502
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657153707.122, "dur": 0.428,
    "args": {
      "External id": 2485704,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2503
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657153709.144, "dur": 0.448,
    "args": {
      "External id": 2485705,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2504
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657153711.147, "dur": 0.632,
    "args": {
      "External id": 2485706,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2505
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657153726.933, "dur": 34.707,
    "args": {
      "External id": 2485707,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2506
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1381179,
    "ts": 1514657153793.349, "dur": 103.882,
    "args": {
      "External id": 2485708,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "1", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 2507
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657153803.972, "dur": 3.234,
    "args": {
      "External id": 2485709,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2508
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1381179,
    "ts": 1514657153812.586, "dur": 10.590,
    "args": {
      "External id": 2485710,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2509
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1381179,
    "ts": 1514657153817.093, "dur": 5.660,
    "args": {
      "External id": 2485711,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "12846080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 2510
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657153820.671, "dur": 1.030,
    "args": {
      "External id": 2485712,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2511
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1381179,
    "ts": 1514657153830.423, "dur": 28.400,
    "args": {
      "External id": 2485713,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2512
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657153833.034, "dur": 0.536,
    "args": {
      "External id": 2485714,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2513
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657153835.709, "dur": 0.378,
    "args": {
      "External id": 2485715,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6423296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2514
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657153837.889, "dur": 2.511,
    "args": {
      "External id": 2485716,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6947584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2515
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657153841.975, "dur": 0.352,
    "args": {
      "External id": 2485717,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7471872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2516
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657153844.391, "dur": 0.576,
    "args": {
      "External id": 2485718,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7996160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2517
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657153846.894, "dur": 0.429,
    "args": {
      "External id": 2485719,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8520448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2518
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657153849.197, "dur": 0.357,
    "args": {
      "External id": 2485720,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "8520704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2519
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657153851.257, "dur": 0.419,
    "args": {
      "External id": 2485721,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "9962496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2520
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657153853.266, "dur": 0.622,
    "args": {
      "External id": 2485722,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11404288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2521
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657153868.917, "dur": 21.350,
    "args": {
      "External id": 2485723,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2522
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657153941.448, "dur": 154.507,
    "args": {
      "External id": 2485724,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 2523
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657153962.637, "dur": 129.291,
    "args": {
      "External id": 2485725,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2524, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514657153972.306, "dur": 115.059,
    "args": {
      "External id": 2485726,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 2525
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657154110.815, "dur": 2.130,
    "args": {
      "External id": 2485727,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2526, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657154196.631, "dur": 1720.268,
    "args": {
      "External id": 2485728,"Sequence number": 24740464, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 2527
    }
  },
  {
    "ph": "f", "id": 184, "pid": 1336754, "tid": 1381179, "ts": 1514657154196.631,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657154310.296, "dur": 110.316,
    "args": {
      "External id": 2485729,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 2528
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514657154457.264, "dur": 43.257,
    "args": {
      "External id": 2485730,"kernel_hash": "cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/fu/cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 2529
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657154518.883, "dur": 73.034,
    "args": {
      "External id": 2485731,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 2530
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657154607.022, "dur": 37.512,
    "args": {
      "External id": 2485732,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2531
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657154652.106, "dur": 47.375,
    "args": {
      "External id": 2485733,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2532
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657154708.614, "dur": 28.991,
    "args": {
      "External id": 2485734,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2533
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657154745.431, "dur": 43.363,
    "args": {
      "External id": 2485735,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2534
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657154815.251, "dur": 24.594,
    "args": {
      "External id": 2485736,"kernel_hash": "cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/wm/cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 2535
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514657154859.457, "dur": 30.903,
    "args": {
      "External id": 2485737,"kernel_hash": "cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/br/cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2536
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657154909.219, "dur": 21.024,
    "args": {
      "External id": 2485738,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 2537
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514657154943.286, "dur": 19.611,
    "args": {
      "External id": 2485739,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 2538
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657154973.393, "dur": 73.401,
    "args": {
      "External id": 2485740,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2539
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657155052.878, "dur": 38.872,
    "args": {
      "External id": 2485741,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2540
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657155123.120, "dur": 180.718,
    "args": {
      "External id": 2485742,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 2541
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657155205.247, "dur": 6.643,
    "args": {
      "External id": 2485743,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2542
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657155214.042, "dur": 2.244,
    "args": {
      "External id": 2485744,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2543
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657155362.339, "dur": 28.225,
    "args": {
      "External id": 2485745,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2544
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657155403.999, "dur": 16.703,
    "args": {
      "External id": 2485746,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2545
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657155429.715, "dur": 50.640,
    "args": {
      "External id": 2485747,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2546
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657155493.127, "dur": 58.525,
    "args": {
      "External id": 2485748,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2547
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657155567.866, "dur": 26.296,
    "args": {
      "External id": 2485749,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2548
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657155599.810, "dur": 34.294,
    "args": {
      "External id": 2485750,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2549
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657155641.339, "dur": 21.732,
    "args": {
      "External id": 2485751,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2550
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657155671.811, "dur": 31.623,
    "args": {
      "External id": 2485752,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2551
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336754, "tid": 1381179,
    "ts": 1514657155726.131, "dur": 25.913,
    "args": {
      "External id": 2485753,"kernel_hash": "c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/6w/c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 2552
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657155770.043, "dur": 24.553,
    "args": {
      "External id": 2485754,"kernel_hash": "c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/4k/c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2553
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657155809.324, "dur": 18.924,
    "args": {
      "External id": 2485755,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 2554
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514657155843.225, "dur": 15.582,
    "args": {
      "External id": 2485756,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 2555
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336754, "tid": 1381179,
    "ts": 1514657155871.031, "dur": 18.072,
    "args": {
      "External id": 2485757,"kernel_hash": "cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/eb/cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 2556
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657155961.392, "dur": 80.729,
    "args": {
      "External id": 2485758,"Record function id": 0, "Ev Idx": 2557
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657155964.728, "dur": 75.578,
    "args": {
      "External id": 2485759,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2558
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657155968.946, "dur": 69.510,
    "args": {
      "External id": 2485760,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2559
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657156033.046, "dur": 4.981,
    "args": {
      "External id": 2485761,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2560
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657156049.672, "dur": 9.353,
    "args": {
      "External id": 2485762,"Record function id": 0, "Ev Idx": 2561
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657156051.644, "dur": 6.932,
    "args": {
      "External id": 2485763,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2562
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657156053.322, "dur": 4.725,
    "args": {
      "External id": 2485764,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2563
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657156054.166, "dur": 3.768,
    "args": {
      "External id": 2485765,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2564
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657156062.204, "dur": 4.083,
    "args": {
      "External id": 2485766,"Record function id": 0, "Ev Idx": 2565
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657156063.366, "dur": 2.501,
    "args": {
      "External id": 2485767,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2566
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657156064.023, "dur": 1.411,
    "args": {
      "External id": 2485768,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2567
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657156064.385, "dur": 0.963,
    "args": {
      "External id": 2485769,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2568
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657156069.456, "dur": 4.448,
    "args": {
      "External id": 2485770,"Record function id": 0, "Ev Idx": 2569
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657156070.830, "dur": 2.551,
    "args": {
      "External id": 2485771,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2570
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657156071.568, "dur": 1.426,
    "args": {
      "External id": 2485772,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2571
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657156072.187, "dur": 0.706,
    "args": {
      "External id": 2485773,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2572
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657156076.919, "dur": 4.259,
    "args": {
      "External id": 2485774,"Record function id": 0, "Ev Idx": 2573
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657156078.541, "dur": 2.246,
    "args": {
      "External id": 2485775,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2574
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657156079.033, "dur": 1.338,
    "args": {
      "External id": 2485776,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2575
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657156079.581, "dur": 0.724,
    "args": {
      "External id": 2485777,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2576
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657156084.271, "dur": 4.084,
    "args": {
      "External id": 2485778,"Record function id": 0, "Ev Idx": 2577
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657156085.664, "dur": 2.309,
    "args": {
      "External id": 2485779,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2578
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657156086.327, "dur": 1.217,
    "args": {
      "External id": 2485780,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2579
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657156086.784, "dur": 0.667,
    "args": {
      "External id": 2485781,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2580
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657156091.506, "dur": 3.511,
    "args": {
      "External id": 2485782,"Record function id": 0, "Ev Idx": 2581
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657156092.644, "dur": 1.986,
    "args": {
      "External id": 2485783,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2582
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657156093.102, "dur": 1.003,
    "args": {
      "External id": 2485784,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2583
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657156093.387, "dur": 0.643,
    "args": {
      "External id": 2485785,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2584
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657156098.530, "dur": 5.287,
    "args": {
      "External id": 2485786,"Record function id": 0, "Ev Idx": 2585
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657156099.628, "dur": 3.770,
    "args": {
      "External id": 2485787,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2586
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657156100.375, "dur": 2.600,
    "args": {
      "External id": 2485788,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2587
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657156102.224, "dur": 0.665,
    "args": {
      "External id": 2485789,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2588
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657156107.035, "dur": 6.336,
    "args": {
      "External id": 2485790,"Record function id": 0, "Ev Idx": 2589
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657156108.305, "dur": 4.685,
    "args": {
      "External id": 2485791,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2590
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657156108.927, "dur": 3.443,
    "args": {
      "External id": 2485792,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2591
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657156109.508, "dur": 2.803,
    "args": {
      "External id": 2485793,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2592
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657156117.381, "dur": 39201.938,
    "args": {
      "External id": 2485794,"Record function id": 0, "Sequence number": 24740463, "Fwd thread id": 1, "Ev Idx": 2593
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657156118.458, "dur": 39191.214,
    "args": {
      "External id": 2485795,"Sequence number": 24740463, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 2594
    }
  },
  {
    "ph": "f", "id": 185, "pid": 1336754, "tid": 1381179, "ts": 1514657156118.458,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.21)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657156152.992, "dur": 40.330,
    "args": {
      "External id": 2485796,"Record function id": 0, "Ev Idx": 2595
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.21)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657156201.605, "dur": 68.451,
    "args": {
      "External id": 2485797,"Record function id": 0, "Ev Idx": 2596
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.21)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657156276.508, "dur": 39024.204,
    "args": {
      "External id": 2485798,"Record function id": 0, "Ev Idx": 2597
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657156364.718, "dur": 6.963,
    "args": {
      "External id": 2485799,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2598
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657156381.205, "dur": 4.550,
    "args": {
      "External id": 2485800,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2599
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514657156404.254, "dur": 38018.597,
    "args": {
      "External id": 2485801,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2600
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514657156417.799, "dur": 37996.432,
    "args": {
      "External id": 2485802,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2601
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657156460.605, "dur": 14.985,
    "args": {
      "External id": 2485803,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2602
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514657156482.148, "dur": 37890.390,
    "args": {
      "External id": 2485804,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 2603
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657156484.533, "dur": 37887.346,
    "args": {
      "External id": 2485805,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 2604
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657156488.524, "dur": 4.834,
    "args": {
      "External id": 2485806,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2605
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657156494.940, "dur": 37873.422,
    "args": {
      "External id": 2485807,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 2606
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657194512.659, "dur": 9.148,
    "args": {
      "External id": 2485808,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 2607
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657194515.766, "dur": 5.612,
    "args": {
      "External id": 2485809,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2608
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657194565.405, "dur": 389.398,
    "args": {
      "External id": 2485810,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 2609
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657194592.091, "dur": 357.270,
    "args": {
      "External id": 2485811,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2610, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514657194604.032, "dur": 339.262,
    "args": {
      "External id": 2485812,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 2611
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657194976.521, "dur": 2.568,
    "args": {
      "External id": 2485813,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2612, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657195075.454, "dur": 7.101,
    "args": {
      "External id": 2485814,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2613
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657195128.877, "dur": 4.252,
    "args": {
      "External id": 2485815,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2614
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657195151.375, "dur": 1.015,
    "args": {
      "External id": 2485816,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2615
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657195165.753, "dur": 1.270,
    "args": {
      "External id": 2485817,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2616
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657195180.117, "dur": 1.098,
    "args": {
      "External id": 2485818,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2617
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657195192.831, "dur": 3.236,
    "args": {
      "External id": 2485819,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2618
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657195208.906, "dur": 0.963,
    "args": {
      "External id": 2485820,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2619
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657195224.899, "dur": 1.834,
    "args": {
      "External id": 2485821,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2620
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657195238.647, "dur": 0.738,
    "args": {
      "External id": 2485822,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2621
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657195339.839, "dur": 2894.472,
    "args": {
      "External id": 2485823,"Record function id": 0, "Ev Idx": 2622
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.20)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657195359.464, "dur": 1101.137,
    "args": {
      "External id": 2485824,"Record function id": 0, "Ev Idx": 2623
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.20)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657195374.511, "dur": 364.078,
    "args": {
      "External id": 2485825,"Record function id": 0, "Ev Idx": 2624
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657195465.283, "dur": 6.213,
    "args": {
      "External id": 2485826,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2625
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657195475.016, "dur": 1.275,
    "args": {
      "External id": 2485827,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2626
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657195478.435, "dur": 1.295,
    "args": {
      "External id": 2485828,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2627
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657195481.584, "dur": 0.869,
    "args": {
      "External id": 2485829,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2628
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657195483.984, "dur": 0.855,
    "args": {
      "External id": 2485830,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2629
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657195486.748, "dur": 1.074,
    "args": {
      "External id": 2485831,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2630
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657195489.419, "dur": 2.095,
    "args": {
      "External id": 2485832,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2631
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657195492.992, "dur": 0.778,
    "args": {
      "External id": 2485833,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2632
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657195495.190, "dur": 3.094,
    "args": {
      "External id": 2485834,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2633
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657195499.859, "dur": 4.147,
    "args": {
      "External id": 2485835,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2634
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657195524.404, "dur": 180.316,
    "args": {
      "External id": 2485836,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2635
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657195557.946, "dur": 141.710,
    "args": {
      "External id": 2485837,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2636
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657195580.421, "dur": 14.691,
    "args": {
      "External id": 2485838,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2637
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514657195598.334, "dur": 71.013,
    "args": {
      "External id": 2485839,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 2638
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657195601.456, "dur": 67.500,
    "args": {
      "External id": 2485840,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 2639
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657195605.828, "dur": 6.223,
    "args": {
      "External id": 2485841,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2640
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657195614.032, "dur": 54.381,
    "args": {
      "External id": 2485842,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 2641
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.19", "pid": 1336754, "tid": 1381179,
    "ts": 1514657195830.577, "dur": 621.702,
    "args": {
      "External id": 2485843,"Record function id": 0, "Ev Idx": 2642
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.19)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657195851.482, "dur": 584.049,
    "args": {
      "External id": 2485844,"Record function id": 0, "Ev Idx": 2643
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657195915.732, "dur": 4.770,
    "args": {
      "External id": 2485845,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2644
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1381179,
    "ts": 1514657195937.105, "dur": 35.462,
    "args": {
      "External id": 2485846,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2645
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657195942.189, "dur": 1.539,
    "args": {
      "External id": 2485847,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2646
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657195946.199, "dur": 0.700,
    "args": {
      "External id": 2485848,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2647
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657195948.666, "dur": 0.408,
    "args": {
      "External id": 2485849,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2648
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657195951.014, "dur": 5.902,
    "args": {
      "External id": 2485850,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2649
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657195958.821, "dur": 0.444,
    "args": {
      "External id": 2485851,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2650
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657195961.020, "dur": 0.439,
    "args": {
      "External id": 2485852,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2651
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657195963.070, "dur": 0.427,
    "args": {
      "External id": 2485853,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2652
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657195965.280, "dur": 0.473,
    "args": {
      "External id": 2485854,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2653
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657195967.547, "dur": 0.413,
    "args": {
      "External id": 2485855,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2654
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657196022.899, "dur": 38.627,
    "args": {
      "External id": 2485856,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2655
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1381179,
    "ts": 1514657196097.667, "dur": 110.467,
    "args": {
      "External id": 2485857,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "1", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 2656
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657196108.821, "dur": 4.523,
    "args": {
      "External id": 2485858,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2657
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1381179,
    "ts": 1514657196118.798, "dur": 10.847,
    "args": {
      "External id": 2485859,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2658
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1381179,
    "ts": 1514657196122.987, "dur": 6.245,
    "args": {
      "External id": 2485860,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "12846080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 2659
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657196127.166, "dur": 0.669,
    "args": {
      "External id": 2485861,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2660
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1381179,
    "ts": 1514657196137.153, "dur": 29.977,
    "args": {
      "External id": 2485862,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2661
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657196140.172, "dur": 2.555,
    "args": {
      "External id": 2485863,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2662
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657196144.818, "dur": 0.455,
    "args": {
      "External id": 2485864,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6423296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2663
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657196146.907, "dur": 0.484,
    "args": {
      "External id": 2485865,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6947584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2664
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657196149.051, "dur": 0.363,
    "args": {
      "External id": 2485866,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7471872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2665
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657196151.302, "dur": 0.494,
    "args": {
      "External id": 2485867,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7996160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2666
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657196153.669, "dur": 0.642,
    "args": {
      "External id": 2485868,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8520448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2667
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657196156.226, "dur": 0.445,
    "args": {
      "External id": 2485869,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "8520704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2668
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657196158.643, "dur": 0.438,
    "args": {
      "External id": 2485870,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "9962496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2669
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657196161.100, "dur": 2.250,
    "args": {
      "External id": 2485871,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11404288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2670
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657196179.436, "dur": 21.314,
    "args": {
      "External id": 2485872,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2671
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657196256.319, "dur": 113.031,
    "args": {
      "External id": 2485873,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 2672
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657196279.144, "dur": 86.526,
    "args": {
      "External id": 2485874,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2673, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514657196289.173, "dur": 71.995,
    "args": {
      "External id": 2485875,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 2674
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657196383.207, "dur": 2.026,
    "args": {
      "External id": 2485876,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2675, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657196467.499, "dur": 1744.797,
    "args": {
      "External id": 2485877,"Sequence number": 24740462, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 2676
    }
  },
  {
    "ph": "f", "id": 186, "pid": 1336754, "tid": 1381179, "ts": 1514657196467.499,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657196592.992, "dur": 107.738,
    "args": {
      "External id": 2485878,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 2677
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514657196739.679, "dur": 44.342,
    "args": {
      "External id": 2485879,"kernel_hash": "cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/fu/cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 2678
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657196805.249, "dur": 51.864,
    "args": {
      "External id": 2485880,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 2679
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657196867.321, "dur": 33.794,
    "args": {
      "External id": 2485881,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2680
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657196908.597, "dur": 47.981,
    "args": {
      "External id": 2485882,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2681
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657196965.490, "dur": 68.918,
    "args": {
      "External id": 2485883,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2682
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657197049.543, "dur": 51.247,
    "args": {
      "External id": 2485884,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2683
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657197127.147, "dur": 28.393,
    "args": {
      "External id": 2485885,"kernel_hash": "cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/wm/cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 2684
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514657197177.975, "dur": 31.096,
    "args": {
      "External id": 2485886,"kernel_hash": "cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/br/cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2685
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657197229.483, "dur": 20.965,
    "args": {
      "External id": 2485887,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 2686
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514657197263.686, "dur": 17.174,
    "args": {
      "External id": 2485888,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 2687
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657197293.206, "dur": 31.758,
    "args": {
      "External id": 2485889,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2688
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657197328.703, "dur": 34.382,
    "args": {
      "External id": 2485890,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2689
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657197390.874, "dur": 199.577,
    "args": {
      "External id": 2485891,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 2690
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657197471.561, "dur": 6.548,
    "args": {
      "External id": 2485892,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2691
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657197480.113, "dur": 2.510,
    "args": {
      "External id": 2485893,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2692
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657197626.292, "dur": 27.499,
    "args": {
      "External id": 2485894,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2693
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657197664.861, "dur": 17.094,
    "args": {
      "External id": 2485895,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2694
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657197691.842, "dur": 41.825,
    "args": {
      "External id": 2485896,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2695
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657197743.392, "dur": 50.679,
    "args": {
      "External id": 2485897,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2696
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657197807.923, "dur": 28.537,
    "args": {
      "External id": 2485898,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2697
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657197843.008, "dur": 36.605,
    "args": {
      "External id": 2485899,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2698
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657197886.642, "dur": 21.286,
    "args": {
      "External id": 2485900,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2699
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657197917.680, "dur": 36.858,
    "args": {
      "External id": 2485901,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2700
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336754, "tid": 1381179,
    "ts": 1514657197973.389, "dur": 60.664,
    "args": {
      "External id": 2485902,"kernel_hash": "c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/6w/c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 2701
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657198055.810, "dur": 25.315,
    "args": {
      "External id": 2485903,"kernel_hash": "c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/4k/c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2702
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657198100.481, "dur": 19.802,
    "args": {
      "External id": 2485904,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 2703
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514657198136.732, "dur": 16.383,
    "args": {
      "External id": 2485905,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 2704
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336754, "tid": 1381179,
    "ts": 1514657198166.299, "dur": 18.312,
    "args": {
      "External id": 2485906,"kernel_hash": "cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/eb/cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 2705
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657198256.404, "dur": 17.923,
    "args": {
      "External id": 2485907,"Record function id": 0, "Ev Idx": 2706
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657198260.151, "dur": 13.203,
    "args": {
      "External id": 2485908,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2707
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657198264.373, "dur": 8.130,
    "args": {
      "External id": 2485909,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2708
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657198266.487, "dur": 5.894,
    "args": {
      "External id": 2485910,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2709
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657198278.094, "dur": 5.188,
    "args": {
      "External id": 2485911,"Record function id": 0, "Ev Idx": 2710
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657198279.793, "dur": 3.055,
    "args": {
      "External id": 2485912,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2711
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657198280.664, "dur": 1.713,
    "args": {
      "External id": 2485913,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2712
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657198281.312, "dur": 0.975,
    "args": {
      "External id": 2485914,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2713
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657198286.740, "dur": 5.486,
    "args": {
      "External id": 2485915,"Record function id": 0, "Ev Idx": 2714
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657198288.616, "dur": 3.166,
    "args": {
      "External id": 2485916,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2715
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657198289.301, "dur": 1.859,
    "args": {
      "External id": 2485917,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2716
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657198289.746, "dur": 1.327,
    "args": {
      "External id": 2485918,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2717
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657198296.152, "dur": 4.347,
    "args": {
      "External id": 2485919,"Record function id": 0, "Ev Idx": 2718
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657198297.521, "dur": 2.600,
    "args": {
      "External id": 2485920,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2719
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657198298.329, "dur": 1.250,
    "args": {
      "External id": 2485921,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2720
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657198298.782, "dur": 0.715,
    "args": {
      "External id": 2485922,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2721
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657198303.972, "dur": 4.203,
    "args": {
      "External id": 2485923,"Record function id": 0, "Ev Idx": 2722
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657198305.332, "dur": 2.437,
    "args": {
      "External id": 2485924,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2723
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657198306.259, "dur": 1.078,
    "args": {
      "External id": 2485925,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2724
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657198306.601, "dur": 0.661,
    "args": {
      "External id": 2485926,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2725
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657198311.698, "dur": 4.158,
    "args": {
      "External id": 2485927,"Record function id": 0, "Ev Idx": 2726
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657198312.992, "dur": 2.483,
    "args": {
      "External id": 2485928,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2727
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657198313.513, "dur": 1.317,
    "args": {
      "External id": 2485929,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2728
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657198314.089, "dur": 0.625,
    "args": {
      "External id": 2485930,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2729
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657198319.041, "dur": 4.498,
    "args": {
      "External id": 2485931,"Record function id": 0, "Ev Idx": 2730
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657198320.611, "dur": 2.513,
    "args": {
      "External id": 2485932,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2731
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657198321.361, "dur": 1.132,
    "args": {
      "External id": 2485933,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2732
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657198321.646, "dur": 0.751,
    "args": {
      "External id": 2485934,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2733
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657198326.707, "dur": 5.680,
    "args": {
      "External id": 2485935,"Record function id": 0, "Ev Idx": 2734
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657198328.101, "dur": 3.889,
    "args": {
      "External id": 2485936,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2735
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657198328.831, "dur": 2.762,
    "args": {
      "External id": 2485937,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2736
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657198329.084, "dur": 2.447,
    "args": {
      "External id": 2485938,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2737
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657198335.739, "dur": 4.671,
    "args": {
      "External id": 2485939,"Record function id": 0, "Ev Idx": 2738
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657198337.043, "dur": 2.967,
    "args": {
      "External id": 2485940,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2739
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657198337.809, "dur": 1.608,
    "args": {
      "External id": 2485941,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2740
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657198338.284, "dur": 1.048,
    "args": {
      "External id": 2485942,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2741
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657198344.284, "dur": 35543.556,
    "args": {
      "External id": 2485943,"Record function id": 0, "Sequence number": 24740461, "Fwd thread id": 1, "Ev Idx": 2742
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657198345.740, "dur": 35533.652,
    "args": {
      "External id": 2485944,"Sequence number": 24740461, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 2743
    }
  },
  {
    "ph": "f", "id": 187, "pid": 1336754, "tid": 1381179, "ts": 1514657198345.740,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.20)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657198374.431, "dur": 35.905,
    "args": {
      "External id": 2485945,"Record function id": 0, "Ev Idx": 2744
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.20)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657198418.047, "dur": 65.041,
    "args": {
      "External id": 2485946,"Record function id": 0, "Ev Idx": 2745
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.20)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657198489.061, "dur": 35383.199,
    "args": {
      "External id": 2485947,"Record function id": 0, "Ev Idx": 2746
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657198594.085, "dur": 7.274,
    "args": {
      "External id": 2485948,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2747
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657198612.437, "dur": 4.940,
    "args": {
      "External id": 2485949,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2748
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514657198633.389, "dur": 34462.693,
    "args": {
      "External id": 2485950,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2749
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514657198647.777, "dur": 34438.935,
    "args": {
      "External id": 2485951,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2750
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657198726.353, "dur": 17.647,
    "args": {
      "External id": 2485952,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2751
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514657198750.507, "dur": 34292.621,
    "args": {
      "External id": 2485953,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 2752
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657198753.155, "dur": 34289.327,
    "args": {
      "External id": 2485954,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 2753
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657198757.152, "dur": 5.116,
    "args": {
      "External id": 2485955,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2754
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657198764.325, "dur": 34274.744,
    "args": {
      "External id": 2485956,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 2755
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657233186.317, "dur": 9.075,
    "args": {
      "External id": 2485957,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 2756
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657233189.753, "dur": 5.272,
    "args": {
      "External id": 2485958,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2757
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657233229.839, "dur": 323.482,
    "args": {
      "External id": 2485959,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 2758
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657233259.649, "dur": 288.024,
    "args": {
      "External id": 2485960,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2759, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514657233271.087, "dur": 269.334,
    "args": {
      "External id": 2485961,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 2760
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657233577.902, "dur": 2.563,
    "args": {
      "External id": 2485962,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2761, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657233641.856, "dur": 6.357,
    "args": {
      "External id": 2485963,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2762
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657233700.955, "dur": 1.620,
    "args": {
      "External id": 2485964,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2763
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657233719.954, "dur": 1.189,
    "args": {
      "External id": 2485965,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2764
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657233734.284, "dur": 1.255,
    "args": {
      "External id": 2485966,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2765
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657233749.166, "dur": 0.977,
    "args": {
      "External id": 2485967,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2766
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657233764.897, "dur": 1.258,
    "args": {
      "External id": 2485968,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2767
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657233779.701, "dur": 1.260,
    "args": {
      "External id": 2485969,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2768
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657233793.819, "dur": 2.354,
    "args": {
      "External id": 2485970,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2769
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657233809.681, "dur": 0.991,
    "args": {
      "External id": 2485971,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2770
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657233902.824, "dur": 2870.919,
    "args": {
      "External id": 2485972,"Record function id": 0, "Ev Idx": 2771
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.19)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657233920.860, "dur": 1052.982,
    "args": {
      "External id": 2485973,"Record function id": 0, "Ev Idx": 2772
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.19)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657233936.040, "dur": 364.829,
    "args": {
      "External id": 2485974,"Record function id": 0, "Ev Idx": 2773
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657234051.406, "dur": 4.560,
    "args": {
      "External id": 2485975,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2774
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657234059.759, "dur": 1.569,
    "args": {
      "External id": 2485976,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2775
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657234063.054, "dur": 1.119,
    "args": {
      "External id": 2485977,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2776
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657234066.061, "dur": 1.223,
    "args": {
      "External id": 2485978,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2777
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657234068.759, "dur": 1.203,
    "args": {
      "External id": 2485979,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2778
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657234078.117, "dur": 1.298,
    "args": {
      "External id": 2485980,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2779
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657234081.230, "dur": 2.129,
    "args": {
      "External id": 2485981,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2780
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657234084.824, "dur": 3.973,
    "args": {
      "External id": 2485982,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2781
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657234090.246, "dur": 0.844,
    "args": {
      "External id": 2485983,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2782
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657234092.552, "dur": 0.999,
    "args": {
      "External id": 2485984,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2783
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657234116.313, "dur": 153.119,
    "args": {
      "External id": 2485985,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2784
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657234132.199, "dur": 132.386,
    "args": {
      "External id": 2485986,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2785
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657234149.373, "dur": 15.646,
    "args": {
      "External id": 2485987,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2786
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514657234168.069, "dur": 68.362,
    "args": {
      "External id": 2485988,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 2787
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657234170.666, "dur": 65.436,
    "args": {
      "External id": 2485989,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 2788
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657234174.889, "dur": 5.844,
    "args": {
      "External id": 2485990,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2789
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657234182.478, "dur": 53.033,
    "args": {
      "External id": 2485991,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 2790
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.18", "pid": 1336754, "tid": 1381179,
    "ts": 1514657234392.288, "dur": 573.754,
    "args": {
      "External id": 2485992,"Record function id": 0, "Ev Idx": 2791
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.18)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657234408.380, "dur": 544.219,
    "args": {
      "External id": 2485993,"Record function id": 0, "Ev Idx": 2792
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657234473.201, "dur": 4.816,
    "args": {
      "External id": 2485994,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2793
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1381179,
    "ts": 1514657234494.014, "dur": 48.982,
    "args": {
      "External id": 2485995,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2794
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657234499.572, "dur": 1.611,
    "args": {
      "External id": 2485996,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2795
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657234503.462, "dur": 0.538,
    "args": {
      "External id": 2485997,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2796
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657234506.103, "dur": 2.586,
    "args": {
      "External id": 2485998,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2797
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657234510.431, "dur": 0.687,
    "args": {
      "External id": 2485999,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2798
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657234512.935, "dur": 0.374,
    "args": {
      "External id": 2486000,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2799
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657234515.279, "dur": 0.401,
    "args": {
      "External id": 2486001,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2800
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657234517.639, "dur": 0.372,
    "args": {
      "External id": 2486002,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2801
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657234520.021, "dur": 0.378,
    "args": {
      "External id": 2486003,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2802
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657234522.148, "dur": 0.383,
    "args": {
      "External id": 2486004,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2803
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657234555.549, "dur": 34.938,
    "args": {
      "External id": 2486005,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2804
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1381179,
    "ts": 1514657234624.869, "dur": 108.850,
    "args": {
      "External id": 2486006,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "1", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 2805
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657234635.717, "dur": 4.012,
    "args": {
      "External id": 2486007,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2806
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1381179,
    "ts": 1514657234645.658, "dur": 12.776,
    "args": {
      "External id": 2486008,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2807
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1381179,
    "ts": 1514657234649.594, "dur": 8.446,
    "args": {
      "External id": 2486009,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "12846080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 2808
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657234653.798, "dur": 2.760,
    "args": {
      "External id": 2486010,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2809
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1381179,
    "ts": 1514657234666.122, "dur": 28.212,
    "args": {
      "External id": 2486011,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2810
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657234668.638, "dur": 0.461,
    "args": {
      "External id": 2486012,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2811
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657234670.925, "dur": 0.577,
    "args": {
      "External id": 2486013,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6423296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2812
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657234673.238, "dur": 0.433,
    "args": {
      "External id": 2486014,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6947584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2813
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657234675.271, "dur": 0.394,
    "args": {
      "External id": 2486015,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7471872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2814
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657234677.498, "dur": 0.403,
    "args": {
      "External id": 2486016,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7996160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2815
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657234680.916, "dur": 0.735,
    "args": {
      "External id": 2486017,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8520448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2816
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657234683.437, "dur": 0.465,
    "args": {
      "External id": 2486018,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "8520704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2817
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657234685.573, "dur": 2.151,
    "args": {
      "External id": 2486019,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "9962496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2818
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657234689.847, "dur": 0.518,
    "args": {
      "External id": 2486020,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11404288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2819
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657234705.187, "dur": 21.458,
    "args": {
      "External id": 2486021,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2820
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657234782.211, "dur": 109.332,
    "args": {
      "External id": 2486022,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 2821
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657234804.197, "dur": 84.341,
    "args": {
      "External id": 2486023,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2822, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514657234814.042, "dur": 70.177,
    "args": {
      "External id": 2486024,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 2823
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657234904.708, "dur": 1.753,
    "args": {
      "External id": 2486025,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2824, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657235022.070, "dur": 1725.989,
    "args": {
      "External id": 2486026,"Sequence number": 24740460, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 2825
    }
  },
  {
    "ph": "f", "id": 188, "pid": 1336754, "tid": 1381179, "ts": 1514657235022.070,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657235138.201, "dur": 111.755,
    "args": {
      "External id": 2486027,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 2826
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514657235292.034, "dur": 42.415,
    "args": {
      "External id": 2486028,"kernel_hash": "cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/fu/cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 2827
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657235358.397, "dur": 57.105,
    "args": {
      "External id": 2486029,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 2828
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657235426.093, "dur": 34.629,
    "args": {
      "External id": 2486030,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2829
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657235469.119, "dur": 46.436,
    "args": {
      "External id": 2486031,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2830
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657235524.056, "dur": 49.475,
    "args": {
      "External id": 2486032,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2831
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657235591.988, "dur": 45.134,
    "args": {
      "External id": 2486033,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2832
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657235668.052, "dur": 24.359,
    "args": {
      "External id": 2486034,"kernel_hash": "cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/wm/cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 2833
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514657235711.022, "dur": 31.465,
    "args": {
      "External id": 2486035,"kernel_hash": "cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/br/cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2834
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657235766.599, "dur": 20.571,
    "args": {
      "External id": 2486036,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 2835
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514657235800.295, "dur": 19.267,
    "args": {
      "External id": 2486037,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 2836
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657235832.693, "dur": 30.896,
    "args": {
      "External id": 2486038,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2837
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657235867.474, "dur": 35.472,
    "args": {
      "External id": 2486039,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2838
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657235931.193, "dur": 222.111,
    "args": {
      "External id": 2486040,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 2839
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657236049.245, "dur": 7.160,
    "args": {
      "External id": 2486041,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2840
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657236058.856, "dur": 2.437,
    "args": {
      "External id": 2486042,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2841
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657236187.808, "dur": 25.289,
    "args": {
      "External id": 2486043,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2842
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657236224.252, "dur": 17.289,
    "args": {
      "External id": 2486044,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2843
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657236253.032, "dur": 43.303,
    "args": {
      "External id": 2486045,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2844
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657236303.675, "dur": 41.703,
    "args": {
      "External id": 2486046,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2845
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657236353.212, "dur": 23.294,
    "args": {
      "External id": 2486047,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2846
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657236382.517, "dur": 47.330,
    "args": {
      "External id": 2486048,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2847
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657236444.940, "dur": 25.148,
    "args": {
      "External id": 2486049,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2848
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657236478.877, "dur": 34.802,
    "args": {
      "External id": 2486050,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2849
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336754, "tid": 1381179,
    "ts": 1514657236547.205, "dur": 29.957,
    "args": {
      "External id": 2486051,"kernel_hash": "c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/6w/c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 2850
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657236596.801, "dur": 25.555,
    "args": {
      "External id": 2486052,"kernel_hash": "c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/4k/c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2851
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657236637.555, "dur": 18.452,
    "args": {
      "External id": 2486053,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 2852
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514657236671.050, "dur": 15.658,
    "args": {
      "External id": 2486054,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 2853
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336754, "tid": 1381179,
    "ts": 1514657236699.492, "dur": 18.581,
    "args": {
      "External id": 2486055,"kernel_hash": "cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/eb/cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 2854
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657236796.226, "dur": 15.371,
    "args": {
      "External id": 2486056,"Record function id": 0, "Ev Idx": 2855
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657236799.422, "dur": 11.213,
    "args": {
      "External id": 2486057,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2856
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657236803.694, "dur": 5.989,
    "args": {
      "External id": 2486058,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2857
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657236805.368, "dur": 4.188,
    "args": {
      "External id": 2486059,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2858
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657236815.851, "dur": 6.096,
    "args": {
      "External id": 2486060,"Record function id": 0, "Ev Idx": 2859
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657236817.486, "dur": 4.023,
    "args": {
      "External id": 2486061,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2860
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657236818.915, "dur": 2.058,
    "args": {
      "External id": 2486062,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2861
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657236819.782, "dur": 1.075,
    "args": {
      "External id": 2486063,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2862
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657236825.654, "dur": 4.936,
    "args": {
      "External id": 2486064,"Record function id": 0, "Ev Idx": 2863
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657236826.831, "dur": 3.371,
    "args": {
      "External id": 2486065,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2864
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657236827.711, "dur": 2.024,
    "args": {
      "External id": 2486066,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2865
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657236828.360, "dur": 1.286,
    "args": {
      "External id": 2486067,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2866
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657236834.095, "dur": 4.766,
    "args": {
      "External id": 2486068,"Record function id": 0, "Ev Idx": 2867
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657236835.673, "dur": 2.802,
    "args": {
      "External id": 2486069,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2868
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657236836.297, "dur": 1.747,
    "args": {
      "External id": 2486070,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2869
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657236836.999, "dur": 0.961,
    "args": {
      "External id": 2486071,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2870
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657236842.222, "dur": 3.822,
    "args": {
      "External id": 2486072,"Record function id": 0, "Ev Idx": 2871
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657236843.501, "dur": 2.134,
    "args": {
      "External id": 2486073,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2872
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657236843.973, "dur": 1.015,
    "args": {
      "External id": 2486074,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2873
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657236844.254, "dur": 0.661,
    "args": {
      "External id": 2486075,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2874
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657236849.433, "dur": 4.041,
    "args": {
      "External id": 2486076,"Record function id": 0, "Ev Idx": 2875
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657236850.706, "dur": 2.272,
    "args": {
      "External id": 2486077,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2876
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657236851.297, "dur": 1.240,
    "args": {
      "External id": 2486078,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2877
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657236851.729, "dur": 0.692,
    "args": {
      "External id": 2486079,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2878
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657236856.928, "dur": 5.832,
    "args": {
      "External id": 2486080,"Record function id": 0, "Ev Idx": 2879
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657236857.986, "dur": 4.341,
    "args": {
      "External id": 2486081,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2880
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657236858.437, "dur": 3.249,
    "args": {
      "External id": 2486082,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2881
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657236858.741, "dur": 2.860,
    "args": {
      "External id": 2486083,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2882
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657236866.271, "dur": 4.097,
    "args": {
      "External id": 2486084,"Record function id": 0, "Ev Idx": 2883
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657236867.420, "dur": 2.547,
    "args": {
      "External id": 2486085,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2884
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657236868.260, "dur": 1.248,
    "args": {
      "External id": 2486086,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2885
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657236868.550, "dur": 0.869,
    "args": {
      "External id": 2486087,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2886
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657236874.033, "dur": 3.587,
    "args": {
      "External id": 2486088,"Record function id": 0, "Ev Idx": 2887
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657236875.100, "dur": 2.105,
    "args": {
      "External id": 2486089,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2888
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657236875.517, "dur": 1.263,
    "args": {
      "External id": 2486090,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2889
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657236875.912, "dur": 0.782,
    "args": {
      "External id": 2486091,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2890
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657236882.089, "dur": 36593.136,
    "args": {
      "External id": 2486092,"Record function id": 0, "Sequence number": 24740459, "Fwd thread id": 1, "Ev Idx": 2891
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657236883.574, "dur": 36582.682,
    "args": {
      "External id": 2486093,"Sequence number": 24740459, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 2892
    }
  },
  {
    "ph": "f", "id": 189, "pid": 1336754, "tid": 1381179, "ts": 1514657236883.574,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.19)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657236912.647, "dur": 38.056,
    "args": {
      "External id": 2486094,"Record function id": 0, "Ev Idx": 2893
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.19)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657236958.027, "dur": 105.477,
    "args": {
      "External id": 2486095,"Record function id": 0, "Ev Idx": 2894
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.19)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657237072.370, "dur": 36386.165,
    "args": {
      "External id": 2486096,"Record function id": 0, "Ev Idx": 2895
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657237162.578, "dur": 7.087,
    "args": {
      "External id": 2486097,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2896
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657237180.364, "dur": 4.872,
    "args": {
      "External id": 2486098,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2897
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514657237199.924, "dur": 35430.071,
    "args": {
      "External id": 2486099,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2898
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514657237213.316, "dur": 35406.821,
    "args": {
      "External id": 2486100,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2899
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657237298.279, "dur": 18.284,
    "args": {
      "External id": 2486101,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2900
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514657237323.152, "dur": 35255.740,
    "args": {
      "External id": 2486102,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 2901
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657237325.630, "dur": 35252.536,
    "args": {
      "External id": 2486103,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 2902
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657237330.377, "dur": 5.453,
    "args": {
      "External id": 2486104,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2903
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657237337.752, "dur": 35236.988,
    "args": {
      "External id": 2486105,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 2904
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657272723.296, "dur": 9.329,
    "args": {
      "External id": 2486106,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 2905
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657272726.745, "dur": 5.591,
    "args": {
      "External id": 2486107,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2906
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657272761.722, "dur": 383.241,
    "args": {
      "External id": 2486108,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 2907
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657272788.230, "dur": 351.112,
    "args": {
      "External id": 2486109,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2908, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514657272799.796, "dur": 333.585,
    "args": {
      "External id": 2486110,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 2909
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657273167.487, "dur": 2.553,
    "args": {
      "External id": 2486111,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2910, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657273231.382, "dur": 8.238,
    "args": {
      "External id": 2486112,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2911
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657273286.346, "dur": 1.600,
    "args": {
      "External id": 2486113,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2912
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657273306.103, "dur": 1.031,
    "args": {
      "External id": 2486114,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2913
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657273321.106, "dur": 0.819,
    "args": {
      "External id": 2486115,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2914
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657273334.823, "dur": 2.511,
    "args": {
      "External id": 2486116,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2915
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657273349.730, "dur": 0.898,
    "args": {
      "External id": 2486117,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2916
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657273364.281, "dur": 0.975,
    "args": {
      "External id": 2486118,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2917
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657273378.455, "dur": 2.466,
    "args": {
      "External id": 2486119,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2918
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657273392.601, "dur": 3.173,
    "args": {
      "External id": 2486120,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2919
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657273490.371, "dur": 2941.333,
    "args": {
      "External id": 2486121,"Record function id": 0, "Ev Idx": 2920
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.18)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657273510.177, "dur": 1136.165,
    "args": {
      "External id": 2486122,"Record function id": 0, "Ev Idx": 2921
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.18)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657273526.347, "dur": 359.566,
    "args": {
      "External id": 2486123,"Record function id": 0, "Ev Idx": 2922
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657273632.274, "dur": 4.778,
    "args": {
      "External id": 2486124,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2923
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657273640.461, "dur": 1.407,
    "args": {
      "External id": 2486125,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2924
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657273643.968, "dur": 1.425,
    "args": {
      "External id": 2486126,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2925
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657273646.935, "dur": 0.894,
    "args": {
      "External id": 2486127,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2926
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657273649.400, "dur": 0.933,
    "args": {
      "External id": 2486128,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2927
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657273652.264, "dur": 0.840,
    "args": {
      "External id": 2486129,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2928
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657273654.690, "dur": 4.019,
    "args": {
      "External id": 2486130,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2929
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657273660.215, "dur": 1.048,
    "args": {
      "External id": 2486131,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2930
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657273662.894, "dur": 1.068,
    "args": {
      "External id": 2486132,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2931
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657273665.826, "dur": 0.782,
    "args": {
      "External id": 2486133,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2932
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657273684.870, "dur": 167.789,
    "args": {
      "External id": 2486134,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2933
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657273701.361, "dur": 146.906,
    "args": {
      "External id": 2486135,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2934
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657273725.056, "dur": 12.485,
    "args": {
      "External id": 2486136,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2935
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514657273740.724, "dur": 74.743,
    "args": {
      "External id": 2486137,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 2936
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657273743.506, "dur": 71.524,
    "args": {
      "External id": 2486138,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 2937
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657273747.629, "dur": 8.462,
    "args": {
      "External id": 2486139,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2938
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657273758.021, "dur": 56.371,
    "args": {
      "External id": 2486140,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 2939
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.17", "pid": 1336754, "tid": 1381179,
    "ts": 1514657273975.090, "dur": 662.554,
    "args": {
      "External id": 2486141,"Record function id": 0, "Ev Idx": 2940
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.17)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657274036.742, "dur": 588.396,
    "args": {
      "External id": 2486142,"Record function id": 0, "Ev Idx": 2941
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657274105.794, "dur": 6.536,
    "args": {
      "External id": 2486143,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2942
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1381179,
    "ts": 1514657274129.269, "dur": 37.065,
    "args": {
      "External id": 2486144,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2943
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657274135.102, "dur": 1.636,
    "args": {
      "External id": 2486145,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2944
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657274139.574, "dur": 2.665,
    "args": {
      "External id": 2486146,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2945
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657274144.251, "dur": 0.403,
    "args": {
      "External id": 2486147,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2946
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657274146.583, "dur": 0.381,
    "args": {
      "External id": 2486148,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2947
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657274148.574, "dur": 0.623,
    "args": {
      "External id": 2486149,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2948
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657274150.810, "dur": 0.375,
    "args": {
      "External id": 2486150,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2949
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657274156.814, "dur": 0.373,
    "args": {
      "External id": 2486151,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2950
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657274159.123, "dur": 0.345,
    "args": {
      "External id": 2486152,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2951
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657274161.323, "dur": 0.398,
    "args": {
      "External id": 2486153,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2952
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657274180.867, "dur": 39.155,
    "args": {
      "External id": 2486154,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2953
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1381179,
    "ts": 1514657274255.179, "dur": 108.689,
    "args": {
      "External id": 2486155,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "1", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 2954
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657274265.654, "dur": 5.544,
    "args": {
      "External id": 2486156,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2955
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1381179,
    "ts": 1514657274276.392, "dur": 10.352,
    "args": {
      "External id": 2486157,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2956
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1381179,
    "ts": 1514657274280.798, "dur": 5.542,
    "args": {
      "External id": 2486158,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "12846080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 2957
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657274284.633, "dur": 0.575,
    "args": {
      "External id": 2486159,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2958
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1381179,
    "ts": 1514657274293.915, "dur": 26.670,
    "args": {
      "External id": 2486160,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2959
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657274296.619, "dur": 0.519,
    "args": {
      "External id": 2486161,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2960
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657274299.146, "dur": 0.417,
    "args": {
      "External id": 2486162,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6423296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2961
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657274301.173, "dur": 0.433,
    "args": {
      "External id": 2486163,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6947584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2962
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657274303.214, "dur": 0.461,
    "args": {
      "External id": 2486164,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7471872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2963
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657274305.169, "dur": 0.414,
    "args": {
      "External id": 2486165,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7996160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2964
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657274307.480, "dur": 0.617,
    "args": {
      "External id": 2486166,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8520448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2965
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657274309.704, "dur": 2.431,
    "args": {
      "External id": 2486167,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "8520704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2966
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657274313.899, "dur": 0.540,
    "args": {
      "External id": 2486168,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "9962496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2967
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657274316.239, "dur": 0.377,
    "args": {
      "External id": 2486169,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11404288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2968
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657274335.266, "dur": 20.958,
    "args": {
      "External id": 2486170,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2969
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657274407.774, "dur": 139.719,
    "args": {
      "External id": 2486171,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 2970
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657274432.760, "dur": 110.731,
    "args": {
      "External id": 2486172,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2971, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514657274446.983, "dur": 76.637,
    "args": {
      "External id": 2486173,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 2972
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657274566.444, "dur": 2.499,
    "args": {
      "External id": 2486174,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2973, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657274654.453, "dur": 1754.145,
    "args": {
      "External id": 2486175,"Sequence number": 24740458, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 2974
    }
  },
  {
    "ph": "f", "id": 190, "pid": 1336754, "tid": 1381179, "ts": 1514657274654.453,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657274769.370, "dur": 106.381,
    "args": {
      "External id": 2486176,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 2975
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514657274913.329, "dur": 42.089,
    "args": {
      "External id": 2486177,"kernel_hash": "cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/fu/cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 2976
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657274975.382, "dur": 98.228,
    "args": {
      "External id": 2486178,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 2977
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657275088.117, "dur": 36.791,
    "args": {
      "External id": 2486179,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2978
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657275132.251, "dur": 47.352,
    "args": {
      "External id": 2486180,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2979
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657275189.901, "dur": 29.475,
    "args": {
      "External id": 2486181,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2980
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657275227.916, "dur": 43.189,
    "args": {
      "External id": 2486182,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2981
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657275296.207, "dur": 26.202,
    "args": {
      "External id": 2486183,"kernel_hash": "cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/wm/cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 2982
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514657275344.676, "dur": 31.923,
    "args": {
      "External id": 2486184,"kernel_hash": "cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/br/cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2983
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657275396.431, "dur": 24.717,
    "args": {
      "External id": 2486185,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 2984
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514657275433.233, "dur": 17.194,
    "args": {
      "External id": 2486186,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 2985
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657275465.866, "dur": 31.543,
    "args": {
      "External id": 2486187,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2986
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657275500.860, "dur": 54.653,
    "args": {
      "External id": 2486188,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2987
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657275585.494, "dur": 183.865,
    "args": {
      "External id": 2486189,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 2988
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657275661.465, "dur": 8.946,
    "args": {
      "External id": 2486190,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2989
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657275672.498, "dur": 4.104,
    "args": {
      "External id": 2486191,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2990
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657275805.744, "dur": 26.683,
    "args": {
      "External id": 2486192,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2991
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657275843.392, "dur": 17.387,
    "args": {
      "External id": 2486193,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2992
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657275870.343, "dur": 60.487,
    "args": {
      "External id": 2486194,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2993
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657275942.664, "dur": 81.295,
    "args": {
      "External id": 2486195,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2994
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276036.660, "dur": 27.943,
    "args": {
      "External id": 2486196,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2995
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276073.307, "dur": 32.988,
    "args": {
      "External id": 2486197,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2996
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276112.803, "dur": 25.234,
    "args": {
      "External id": 2486198,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2997
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276146.206, "dur": 32.465,
    "args": {
      "External id": 2486199,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2998
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276203.037, "dur": 25.404,
    "args": {
      "External id": 2486200,"kernel_hash": "c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/6w/c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 2999
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276246.961, "dur": 28.815,
    "args": {
      "External id": 2486201,"kernel_hash": "c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/4k/c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3000
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276291.792, "dur": 23.130,
    "args": {
      "External id": 2486202,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 3001
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276329.082, "dur": 14.601,
    "args": {
      "External id": 2486203,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 3002
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276358.102, "dur": 22.065,
    "args": {
      "External id": 2486204,"kernel_hash": "cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/eb/cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 3003
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276454.677, "dur": 15.456,
    "args": {
      "External id": 2486205,"Record function id": 0, "Ev Idx": 3004
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276458.207, "dur": 10.908,
    "args": {
      "External id": 2486206,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3005
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276462.427, "dur": 5.941,
    "args": {
      "External id": 2486207,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3006
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276464.359, "dur": 3.880,
    "args": {
      "External id": 2486208,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3007
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276474.464, "dur": 5.635,
    "args": {
      "External id": 2486209,"Record function id": 0, "Ev Idx": 3008
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276475.676, "dur": 3.963,
    "args": {
      "External id": 2486210,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3009
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276476.743, "dur": 2.413,
    "args": {
      "External id": 2486211,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3010
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276477.863, "dur": 1.176,
    "args": {
      "External id": 2486212,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3011
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276486.759, "dur": 5.454,
    "args": {
      "External id": 2486213,"Record function id": 0, "Ev Idx": 3012
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276488.132, "dur": 3.677,
    "args": {
      "External id": 2486214,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3013
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276488.970, "dur": 2.249,
    "args": {
      "External id": 2486215,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3014
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276489.832, "dur": 1.295,
    "args": {
      "External id": 2486216,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3015
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276495.705, "dur": 6.281,
    "args": {
      "External id": 2486217,"Record function id": 0, "Ev Idx": 3016
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276497.045, "dur": 4.547,
    "args": {
      "External id": 2486218,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3017
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276497.641, "dur": 3.529,
    "args": {
      "External id": 2486219,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3018
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276498.038, "dur": 3.048,
    "args": {
      "External id": 2486220,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3019
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276505.323, "dur": 4.420,
    "args": {
      "External id": 2486221,"Record function id": 0, "Ev Idx": 3020
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276506.776, "dur": 2.492,
    "args": {
      "External id": 2486222,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3021
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276507.245, "dur": 1.615,
    "args": {
      "External id": 2486223,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3022
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276507.795, "dur": 0.993,
    "args": {
      "External id": 2486224,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3023
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276512.917, "dur": 5.984,
    "args": {
      "External id": 2486225,"Record function id": 0, "Ev Idx": 3024
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276514.307, "dur": 4.134,
    "args": {
      "External id": 2486226,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3025
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276516.389, "dur": 1.517,
    "args": {
      "External id": 2486227,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3026
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276516.910, "dur": 0.878,
    "args": {
      "External id": 2486228,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3027
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276522.202, "dur": 4.346,
    "args": {
      "External id": 2486229,"Record function id": 0, "Ev Idx": 3028
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276523.676, "dur": 2.468,
    "args": {
      "External id": 2486230,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3029
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276524.289, "dur": 1.417,
    "args": {
      "External id": 2486231,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3030
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276524.922, "dur": 0.692,
    "args": {
      "External id": 2486232,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3031
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276550.560, "dur": 6.207,
    "args": {
      "External id": 2486233,"Record function id": 0, "Ev Idx": 3032
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276552.279, "dur": 3.858,
    "args": {
      "External id": 2486234,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3033
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276553.323, "dur": 2.024,
    "args": {
      "External id": 2486235,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3034
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276553.834, "dur": 1.329,
    "args": {
      "External id": 2486236,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3035
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276559.959, "dur": 4.781,
    "args": {
      "External id": 2486237,"Record function id": 0, "Ev Idx": 3036
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276561.370, "dur": 2.983,
    "args": {
      "External id": 2486238,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3037
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276561.953, "dur": 1.859,
    "args": {
      "External id": 2486239,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3038
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276562.673, "dur": 1.043,
    "args": {
      "External id": 2486240,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3039
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276568.583, "dur": 36146.183,
    "args": {
      "External id": 2486241,"Record function id": 0, "Sequence number": 24740457, "Fwd thread id": 1, "Ev Idx": 3040
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276569.863, "dur": 36136.017,
    "args": {
      "External id": 2486242,"Sequence number": 24740457, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 3041
    }
  },
  {
    "ph": "f", "id": 191, "pid": 1336754, "tid": 1381179, "ts": 1514657276569.863,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.18)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276599.101, "dur": 37.988,
    "args": {
      "External id": 2486243,"Record function id": 0, "Ev Idx": 3042
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.18)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276644.767, "dur": 68.931,
    "args": {
      "External id": 2486244,"Record function id": 0, "Ev Idx": 3043
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.18)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276719.635, "dur": 35976.774,
    "args": {
      "External id": 2486245,"Record function id": 0, "Ev Idx": 3044
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276807.966, "dur": 6.796,
    "args": {
      "External id": 2486246,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3045
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276824.801, "dur": 4.657,
    "args": {
      "External id": 2486247,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3046
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276844.939, "dur": 34954.208,
    "args": {
      "External id": 2486248,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3047
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276892.198, "dur": 34897.157,
    "args": {
      "External id": 2486249,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3048
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276946.924, "dur": 16.315,
    "args": {
      "External id": 2486250,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3049
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276969.711, "dur": 34777.627,
    "args": {
      "External id": 2486251,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 3050
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276972.529, "dur": 34773.733,
    "args": {
      "External id": 2486252,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 3051
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657276976.831, "dur": 5.672,
    "args": {
      "External id": 2486253,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3052
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657277022.020, "dur": 34720.462,
    "args": {
      "External id": 2486254,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 3053
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657311889.499, "dur": 9.427,
    "args": {
      "External id": 2486255,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 3054
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657311893.121, "dur": 5.478,
    "args": {
      "External id": 2486256,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3055
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657311927.284, "dur": 427.995,
    "args": {
      "External id": 2486257,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 3056
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657311953.574, "dur": 395.674,
    "args": {
      "External id": 2486258,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 3057, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514657311964.797, "dur": 377.829,
    "args": {
      "External id": 2486259,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 3058
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657312377.452, "dur": 2.929,
    "args": {
      "External id": 2486260,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 3059, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657312446.225, "dur": 8.054,
    "args": {
      "External id": 2486261,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3060
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657312503.209, "dur": 1.383,
    "args": {
      "External id": 2486262,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3061
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657312523.301, "dur": 1.320,
    "args": {
      "External id": 2486263,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3062
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657312559.404, "dur": 1.635,
    "args": {
      "External id": 2486264,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3063
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657312574.775, "dur": 0.924,
    "args": {
      "External id": 2486265,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3064
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657312587.565, "dur": 0.907,
    "args": {
      "External id": 2486266,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3065
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657312601.781, "dur": 0.860,
    "args": {
      "External id": 2486267,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3066
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657312617.549, "dur": 2.434,
    "args": {
      "External id": 2486268,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3067
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657312632.490, "dur": 1.002,
    "args": {
      "External id": 2486269,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3068
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657312731.026, "dur": 2984.704,
    "args": {
      "External id": 2486270,"Record function id": 0, "Ev Idx": 3069
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.17)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657312749.404, "dur": 1151.813,
    "args": {
      "External id": 2486271,"Record function id": 0, "Ev Idx": 3070
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.17)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657312764.320, "dur": 406.132,
    "args": {
      "External id": 2486272,"Record function id": 0, "Ev Idx": 3071
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657312882.388, "dur": 4.580,
    "args": {
      "External id": 2486273,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3072
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657312890.388, "dur": 1.300,
    "args": {
      "External id": 2486274,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3073
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657312893.439, "dur": 0.963,
    "args": {
      "External id": 2486275,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3074
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657312896.021, "dur": 2.716,
    "args": {
      "External id": 2486276,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3075
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657312900.300, "dur": 1.045,
    "args": {
      "External id": 2486277,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3076
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657312904.961, "dur": 0.781,
    "args": {
      "External id": 2486278,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3077
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657312907.326, "dur": 1.891,
    "args": {
      "External id": 2486279,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3078
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657312910.768, "dur": 0.900,
    "args": {
      "External id": 2486280,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3079
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657312912.988, "dur": 0.762,
    "args": {
      "External id": 2486281,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3080
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657312915.146, "dur": 0.996,
    "args": {
      "External id": 2486282,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3081
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657312933.901, "dur": 197.639,
    "args": {
      "External id": 2486283,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3082
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657312950.581, "dur": 174.967,
    "args": {
      "External id": 2486284,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3083
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657312966.508, "dur": 13.015,
    "args": {
      "External id": 2486285,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3084
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514657313018.048, "dur": 75.261,
    "args": {
      "External id": 2486286,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 3085
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657313020.882, "dur": 71.971,
    "args": {
      "External id": 2486287,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 3086
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657313025.314, "dur": 8.900,
    "args": {
      "External id": 2486288,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3087
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657313036.313, "dur": 55.942,
    "args": {
      "External id": 2486289,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 3088
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.16", "pid": 1336754, "tid": 1381179,
    "ts": 1514657313276.278, "dur": 616.866,
    "args": {
      "External id": 2486290,"Record function id": 0, "Ev Idx": 3089
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.16)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657313295.558, "dur": 581.721,
    "args": {
      "External id": 2486291,"Record function id": 0, "Ev Idx": 3090
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657313360.261, "dur": 5.249,
    "args": {
      "External id": 2486292,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3091
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1381179,
    "ts": 1514657313381.955, "dur": 32.227,
    "args": {
      "External id": 2486293,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 3092
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657313387.315, "dur": 1.598,
    "args": {
      "External id": 2486294,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3093
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657313391.278, "dur": 0.462,
    "args": {
      "External id": 2486295,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3094
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657313393.760, "dur": 0.618,
    "args": {
      "External id": 2486296,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3095
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657313396.077, "dur": 0.531,
    "args": {
      "External id": 2486297,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3096
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657313398.409, "dur": 0.606,
    "args": {
      "External id": 2486298,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3097
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657313401.000, "dur": 0.351,
    "args": {
      "External id": 2486299,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3098
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657313402.971, "dur": 2.747,
    "args": {
      "External id": 2486300,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3099
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657313407.620, "dur": 0.371,
    "args": {
      "External id": 2486301,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3100
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657313409.474, "dur": 0.444,
    "args": {
      "External id": 2486302,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3101
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657313425.084, "dur": 32.961,
    "args": {
      "External id": 2486303,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 3102
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1381179,
    "ts": 1514657313492.260, "dur": 138.920,
    "args": {
      "External id": 2486304,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "1", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 3103
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657313502.436, "dur": 3.293,
    "args": {
      "External id": 2486305,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3104
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1381179,
    "ts": 1514657313510.589, "dur": 10.357,
    "args": {
      "External id": 2486306,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 3105
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1381179,
    "ts": 1514657313514.761, "dur": 5.779,
    "args": {
      "External id": 2486307,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "12846080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 3106
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657313518.645, "dur": 0.801,
    "args": {
      "External id": 2486308,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 3107
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1381179,
    "ts": 1514657313547.646, "dur": 28.471,
    "args": {
      "External id": 2486309,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 3108
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657313550.988, "dur": 0.857,
    "args": {
      "External id": 2486310,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3109
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657313554.186, "dur": 0.447,
    "args": {
      "External id": 2486311,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6423296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3110
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657313556.569, "dur": 0.427,
    "args": {
      "External id": 2486312,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6947584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3111
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657313558.996, "dur": 2.435,
    "args": {
      "External id": 2486313,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7471872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3112
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657313563.091, "dur": 0.446,
    "args": {
      "External id": 2486314,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7996160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3113
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657313565.106, "dur": 0.357,
    "args": {
      "External id": 2486315,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8520448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3114
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657313567.332, "dur": 0.434,
    "args": {
      "External id": 2486316,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "8520704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3115
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657313569.462, "dur": 0.323,
    "args": {
      "External id": 2486317,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "9962496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3116
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657313571.541, "dur": 0.427,
    "args": {
      "External id": 2486318,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11404288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3117
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657313595.390, "dur": 27.817,
    "args": {
      "External id": 2486319,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 3118
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657313682.834, "dur": 124.791,
    "args": {
      "External id": 2486320,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 3119
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657313709.502, "dur": 94.003,
    "args": {
      "External id": 2486321,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 3120, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514657313719.680, "dur": 79.055,
    "args": {
      "External id": 2486322,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 3121
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657313824.905, "dur": 1.822,
    "args": {
      "External id": 2486323,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 3122, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657313909.663, "dur": 1782.903,
    "args": {
      "External id": 2486324,"Sequence number": 24740456, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 3123
    }
  },
  {
    "ph": "f", "id": 192, "pid": 1336754, "tid": 1381179, "ts": 1514657313909.663,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657314061.606, "dur": 107.288,
    "args": {
      "External id": 2486325,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 3124
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514657314212.907, "dur": 45.928,
    "args": {
      "External id": 2486326,"kernel_hash": "cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/fu/cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 3125
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657314276.959, "dur": 53.287,
    "args": {
      "External id": 2486327,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 3126
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657314340.779, "dur": 34.583,
    "args": {
      "External id": 2486328,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 3127
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657314384.972, "dur": 52.956,
    "args": {
      "External id": 2486329,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 3128
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657314446.447, "dur": 29.191,
    "args": {
      "External id": 2486330,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 3129
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657314483.932, "dur": 58.792,
    "args": {
      "External id": 2486331,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 3130
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657314570.795, "dur": 25.687,
    "args": {
      "External id": 2486332,"kernel_hash": "cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/wm/cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 3131
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514657314617.924, "dur": 31.526,
    "args": {
      "External id": 2486333,"kernel_hash": "cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/br/cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3132
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657314670.131, "dur": 20.925,
    "args": {
      "External id": 2486334,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 3133
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514657314703.433, "dur": 16.336,
    "args": {
      "External id": 2486335,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 3134
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657314729.966, "dur": 34.189,
    "args": {
      "External id": 2486336,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3135
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657314767.896, "dur": 35.509,
    "args": {
      "External id": 2486337,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3136
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657314833.002, "dur": 220.551,
    "args": {
      "External id": 2486338,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 3137
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657314914.552, "dur": 9.391,
    "args": {
      "External id": 2486339,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3138
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657314926.656, "dur": 2.391,
    "args": {
      "External id": 2486340,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3139
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657315091.331, "dur": 41.340,
    "args": {
      "External id": 2486341,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3140
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657315168.739, "dur": 18.778,
    "args": {
      "External id": 2486342,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3141
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657315204.151, "dur": 45.693,
    "args": {
      "External id": 2486343,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3142
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657315257.227, "dur": 42.344,
    "args": {
      "External id": 2486344,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3143
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657315308.838, "dur": 26.085,
    "args": {
      "External id": 2486345,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3144
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657315341.350, "dur": 32.291,
    "args": {
      "External id": 2486346,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3145
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657315383.588, "dur": 21.711,
    "args": {
      "External id": 2486347,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3146
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657315414.094, "dur": 34.389,
    "args": {
      "External id": 2486348,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3147
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336754, "tid": 1381179,
    "ts": 1514657315464.680, "dur": 23.287,
    "args": {
      "External id": 2486349,"kernel_hash": "c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/6w/c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 3148
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657315508.514, "dur": 41.645,
    "args": {
      "External id": 2486350,"kernel_hash": "c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/4k/c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3149
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657315572.145, "dur": 21.830,
    "args": {
      "External id": 2486351,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 3150
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514657315614.293, "dur": 16.089,
    "args": {
      "External id": 2486352,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 3151
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336754, "tid": 1381179,
    "ts": 1514657315643.088, "dur": 18.160,
    "args": {
      "External id": 2486353,"kernel_hash": "cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/eb/cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 3152
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657315738.952, "dur": 15.231,
    "args": {
      "External id": 2486354,"Record function id": 0, "Ev Idx": 3153
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657315742.318, "dur": 10.758,
    "args": {
      "External id": 2486355,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3154
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657315746.257, "dur": 5.816,
    "args": {
      "External id": 2486356,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3155
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657315747.947, "dur": 3.985,
    "args": {
      "External id": 2486357,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3156
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657315758.343, "dur": 5.510,
    "args": {
      "External id": 2486358,"Record function id": 0, "Ev Idx": 3157
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657315759.947, "dur": 3.461,
    "args": {
      "External id": 2486359,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3158
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657315760.749, "dur": 2.020,
    "args": {
      "External id": 2486360,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3159
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657315761.562, "dur": 1.083,
    "args": {
      "External id": 2486361,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3160
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657315767.334, "dur": 6.501,
    "args": {
      "External id": 2486362,"Record function id": 0, "Ev Idx": 3161
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657315768.746, "dur": 4.678,
    "args": {
      "External id": 2486363,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3162
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657315769.498, "dur": 3.434,
    "args": {
      "External id": 2486364,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3163
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657315770.037, "dur": 2.806,
    "args": {
      "External id": 2486365,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3164
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657315777.149, "dur": 3.808,
    "args": {
      "External id": 2486366,"Record function id": 0, "Ev Idx": 3165
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657315778.189, "dur": 2.370,
    "args": {
      "External id": 2486367,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3166
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657315778.960, "dur": 1.175,
    "args": {
      "External id": 2486368,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3167
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657315779.263, "dur": 0.798,
    "args": {
      "External id": 2486369,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3168
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657315784.107, "dur": 4.209,
    "args": {
      "External id": 2486370,"Record function id": 0, "Ev Idx": 3169
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657315785.298, "dur": 2.638,
    "args": {
      "External id": 2486371,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3170
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657315786.094, "dur": 1.314,
    "args": {
      "External id": 2486372,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3171
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657315786.599, "dur": 0.746,
    "args": {
      "External id": 2486373,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3172
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657315791.667, "dur": 4.544,
    "args": {
      "External id": 2486374,"Record function id": 0, "Ev Idx": 3173
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657315793.079, "dur": 2.683,
    "args": {
      "External id": 2486375,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3174
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657315793.678, "dur": 1.524,
    "args": {
      "External id": 2486376,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3175
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657315794.221, "dur": 0.885,
    "args": {
      "External id": 2486377,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3176
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657315799.496, "dur": 3.945,
    "args": {
      "External id": 2486378,"Record function id": 0, "Ev Idx": 3177
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657315800.769, "dur": 2.255,
    "args": {
      "External id": 2486379,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3178
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657315801.376, "dur": 1.220,
    "args": {
      "External id": 2486380,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3179
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657315801.961, "dur": 0.556,
    "args": {
      "External id": 2486381,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3180
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657315806.552, "dur": 6.766,
    "args": {
      "External id": 2486382,"Record function id": 0, "Ev Idx": 3181
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657315807.743, "dur": 5.189,
    "args": {
      "External id": 2486383,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3182
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657315808.353, "dur": 4.161,
    "args": {
      "External id": 2486384,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3183
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657315808.636, "dur": 3.813,
    "args": {
      "External id": 2486385,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3184
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657315816.469, "dur": 3.954,
    "args": {
      "External id": 2486386,"Record function id": 0, "Ev Idx": 3185
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657315817.730, "dur": 2.300,
    "args": {
      "External id": 2486387,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3186
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657315818.321, "dur": 1.082,
    "args": {
      "External id": 2486388,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3187
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657315818.740, "dur": 0.573,
    "args": {
      "External id": 2486389,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3188
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657315824.739, "dur": 36210.614,
    "args": {
      "External id": 2486390,"Record function id": 0, "Sequence number": 24740455, "Fwd thread id": 1, "Ev Idx": 3189
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657315826.195, "dur": 36198.782,
    "args": {
      "External id": 2486391,"Sequence number": 24740455, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 3190
    }
  },
  {
    "ph": "f", "id": 193, "pid": 1336754, "tid": 1381179, "ts": 1514657315826.195,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.17)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657315857.504, "dur": 41.667,
    "args": {
      "External id": 2486392,"Record function id": 0, "Ev Idx": 3191
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.17)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657315907.155, "dur": 67.317,
    "args": {
      "External id": 2486393,"Record function id": 0, "Ev Idx": 3192
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.17)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657316017.134, "dur": 35963.973,
    "args": {
      "External id": 2486394,"Record function id": 0, "Ev Idx": 3193
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657316111.907, "dur": 7.466,
    "args": {
      "External id": 2486395,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3194
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657316129.780, "dur": 7.021,
    "args": {
      "External id": 2486396,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3195
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514657316152.614, "dur": 35024.698,
    "args": {
      "External id": 2486397,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3196
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514657316166.571, "dur": 35001.076,
    "args": {
      "External id": 2486398,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3197
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657316223.945, "dur": 14.687,
    "args": {
      "External id": 2486399,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3198
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514657316245.005, "dur": 34882.569,
    "args": {
      "External id": 2486400,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 3199
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657316247.570, "dur": 34879.392,
    "args": {
      "External id": 2486401,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 3200
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657316251.686, "dur": 4.960,
    "args": {
      "External id": 2486402,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3201
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657316258.523, "dur": 34864.764,
    "args": {
      "External id": 2486403,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 3202
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657351265.975, "dur": 9.416,
    "args": {
      "External id": 2486404,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 3203
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657351269.313, "dur": 5.708,
    "args": {
      "External id": 2486405,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3204
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657351305.925, "dur": 366.455,
    "args": {
      "External id": 2486406,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 3205
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657351332.423, "dur": 334.899,
    "args": {
      "External id": 2486407,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 3206, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514657351343.962, "dur": 317.449,
    "args": {
      "External id": 2486408,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 3207
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657351692.948, "dur": 2.465,
    "args": {
      "External id": 2486409,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 3208, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657351754.413, "dur": 6.593,
    "args": {
      "External id": 2486410,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3209
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657351808.794, "dur": 1.300,
    "args": {
      "External id": 2486411,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3210
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657351829.410, "dur": 3.067,
    "args": {
      "External id": 2486412,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3211
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657351847.037, "dur": 0.888,
    "args": {
      "External id": 2486413,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3212
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657351861.616, "dur": 0.877,
    "args": {
      "External id": 2486414,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3213
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657351875.412, "dur": 1.121,
    "args": {
      "External id": 2486415,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3214
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657351889.896, "dur": 2.698,
    "args": {
      "External id": 2486416,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3215
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657351906.032, "dur": 2.467,
    "args": {
      "External id": 2486417,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3216
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657351919.533, "dur": 0.835,
    "args": {
      "External id": 2486418,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3217
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657352052.869, "dur": 2869.779,
    "args": {
      "External id": 2486419,"Record function id": 0, "Ev Idx": 3218
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.16)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657352074.461, "dur": 1097.233,
    "args": {
      "External id": 2486420,"Record function id": 0, "Ev Idx": 3219
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.16)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657352089.382, "dur": 334.351,
    "args": {
      "External id": 2486421,"Record function id": 0, "Ev Idx": 3220
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657352175.790, "dur": 4.533,
    "args": {
      "External id": 2486422,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3221
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657352183.923, "dur": 1.349,
    "args": {
      "External id": 2486423,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3222
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657352187.417, "dur": 3.407,
    "args": {
      "External id": 2486424,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3223
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657352192.469, "dur": 0.899,
    "args": {
      "External id": 2486425,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3224
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657352195.397, "dur": 1.002,
    "args": {
      "External id": 2486426,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3225
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657352198.614, "dur": 0.842,
    "args": {
      "External id": 2486427,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3226
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657352201.054, "dur": 1.953,
    "args": {
      "External id": 2486428,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3227
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657352204.754, "dur": 0.890,
    "args": {
      "External id": 2486429,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3228
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657352207.130, "dur": 0.813,
    "args": {
      "External id": 2486430,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3229
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657352209.618, "dur": 0.816,
    "args": {
      "External id": 2486431,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3230
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657352228.586, "dur": 161.979,
    "args": {
      "External id": 2486432,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3231
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657352246.015, "dur": 139.713,
    "args": {
      "External id": 2486433,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3232
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657352270.477, "dur": 13.919,
    "args": {
      "External id": 2486434,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3233
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514657352287.535, "dur": 68.655,
    "args": {
      "External id": 2486435,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 3234
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657352290.130, "dur": 65.685,
    "args": {
      "External id": 2486436,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 3235
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657352294.458, "dur": 5.131,
    "args": {
      "External id": 2486437,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3236
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657352301.704, "dur": 53.534,
    "args": {
      "External id": 2486438,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 3237
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.15", "pid": 1336754, "tid": 1381179,
    "ts": 1514657352514.754, "dur": 649.261,
    "args": {
      "External id": 2486439,"Record function id": 0, "Ev Idx": 3238
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.15)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657352550.551, "dur": 599.289,
    "args": {
      "External id": 2486440,"Record function id": 0, "Ev Idx": 3239
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657352621.240, "dur": 6.233,
    "args": {
      "External id": 2486441,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3240
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1381179,
    "ts": 1514657352644.372, "dur": 36.532,
    "args": {
      "External id": 2486442,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 3241
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657352649.766, "dur": 1.586,
    "args": {
      "External id": 2486443,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3242
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657352653.781, "dur": 0.564,
    "args": {
      "External id": 2486444,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3243
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657352656.012, "dur": 0.373,
    "args": {
      "External id": 2486445,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3244
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657352661.671, "dur": 0.432,
    "args": {
      "External id": 2486446,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3245
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657352663.685, "dur": 0.840,
    "args": {
      "External id": 2486447,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3246
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657352666.479, "dur": 2.361,
    "args": {
      "External id": 2486448,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3247
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657352670.723, "dur": 0.366,
    "args": {
      "External id": 2486449,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3248
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657352673.330, "dur": 0.387,
    "args": {
      "External id": 2486450,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3249
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657352675.438, "dur": 0.380,
    "args": {
      "External id": 2486451,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3250
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657352692.247, "dur": 38.991,
    "args": {
      "External id": 2486452,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 3251
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1381179,
    "ts": 1514657352763.181, "dur": 103.195,
    "args": {
      "External id": 2486453,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "1", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 3252
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657352773.676, "dur": 3.535,
    "args": {
      "External id": 2486454,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3253
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1381179,
    "ts": 1514657352782.648, "dur": 10.195,
    "args": {
      "External id": 2486455,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 3254
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1381179,
    "ts": 1514657352786.776, "dur": 5.632,
    "args": {
      "External id": 2486456,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "12846080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 3255
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657352790.671, "dur": 0.468,
    "args": {
      "External id": 2486457,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 3256
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1381179,
    "ts": 1514657352800.129, "dur": 26.282,
    "args": {
      "External id": 2486458,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 3257
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657352802.598, "dur": 0.514,
    "args": {
      "External id": 2486459,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3258
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657352804.958, "dur": 0.417,
    "args": {
      "External id": 2486460,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6423296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3259
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657352807.498, "dur": 2.346,
    "args": {
      "External id": 2486461,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6947584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3260
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657352811.628, "dur": 0.424,
    "args": {
      "External id": 2486462,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7471872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3261
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657352814.164, "dur": 0.358,
    "args": {
      "External id": 2486463,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7996160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3262
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657352816.060, "dur": 0.344,
    "args": {
      "External id": 2486464,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8520448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3263
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657352818.019, "dur": 0.406,
    "args": {
      "External id": 2486465,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "8520704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3264
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657352820.089, "dur": 0.423,
    "args": {
      "External id": 2486466,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "9962496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3265
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657352822.279, "dur": 0.483,
    "args": {
      "External id": 2486467,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11404288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3266
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657352838.459, "dur": 20.443,
    "args": {
      "External id": 2486468,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 3267
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657352914.992, "dur": 160.833,
    "args": {
      "External id": 2486469,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 3268
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657352939.658, "dur": 131.964,
    "args": {
      "External id": 2486470,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 3269, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514657352949.557, "dur": 117.471,
    "args": {
      "External id": 2486471,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 3270
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657353091.817, "dur": 2.067,
    "args": {
      "External id": 2486472,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 3271, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657353180.426, "dur": 1721.697,
    "args": {
      "External id": 2486473,"Sequence number": 24740454, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 3272
    }
  },
  {
    "ph": "f", "id": 194, "pid": 1336754, "tid": 1381179, "ts": 1514657353180.426,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657353291.285, "dur": 105.681,
    "args": {
      "External id": 2486474,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 3273
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514657353431.968, "dur": 41.984,
    "args": {
      "External id": 2486475,"kernel_hash": "cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/fu/cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 3274
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657353492.119, "dur": 71.607,
    "args": {
      "External id": 2486476,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 3275
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657353579.813, "dur": 39.601,
    "args": {
      "External id": 2486477,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 3276
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657353627.592, "dur": 47.311,
    "args": {
      "External id": 2486478,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 3277
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657353704.724, "dur": 30.444,
    "args": {
      "External id": 2486479,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 3278
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657353744.553, "dur": 47.534,
    "args": {
      "External id": 2486480,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 3279
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657353817.887, "dur": 24.150,
    "args": {
      "External id": 2486481,"kernel_hash": "cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/wm/cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 3280
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514657353859.875, "dur": 30.291,
    "args": {
      "External id": 2486482,"kernel_hash": "cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/br/cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3281
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657353910.973, "dur": 21.279,
    "args": {
      "External id": 2486483,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 3282
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514657353946.879, "dur": 16.532,
    "args": {
      "External id": 2486484,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 3283
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657353973.843, "dur": 72.851,
    "args": {
      "External id": 2486485,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3284
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657354052.556, "dur": 38.230,
    "args": {
      "External id": 2486486,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3285
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657354121.317, "dur": 187.075,
    "args": {
      "External id": 2486487,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 3286
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657354205.968, "dur": 5.806,
    "args": {
      "External id": 2486488,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3287
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657354213.775, "dur": 2.178,
    "args": {
      "External id": 2486489,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3288
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657354347.554, "dur": 24.383,
    "args": {
      "External id": 2486490,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3289
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657354382.884, "dur": 16.233,
    "args": {
      "External id": 2486491,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3290
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657354408.036, "dur": 36.089,
    "args": {
      "External id": 2486492,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3291
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657354451.196, "dur": 52.200,
    "args": {
      "External id": 2486493,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3292
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657354518.233, "dur": 45.966,
    "args": {
      "External id": 2486494,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3293
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657354572.840, "dur": 39.788,
    "args": {
      "External id": 2486495,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3294
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657354619.790, "dur": 20.875,
    "args": {
      "External id": 2486496,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3295
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657354648.196, "dur": 33.322,
    "args": {
      "External id": 2486497,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3296
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336754, "tid": 1381179,
    "ts": 1514657354702.368, "dur": 27.880,
    "args": {
      "External id": 2486498,"kernel_hash": "c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/6w/c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 3297
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657354746.909, "dur": 27.801,
    "args": {
      "External id": 2486499,"kernel_hash": "c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/4k/c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3298
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657354790.527, "dur": 19.008,
    "args": {
      "External id": 2486500,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 3299
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514657354824.708, "dur": 18.348,
    "args": {
      "External id": 2486501,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 3300
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336754, "tid": 1381179,
    "ts": 1514657354854.503, "dur": 17.873,
    "args": {
      "External id": 2486502,"kernel_hash": "cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/eb/cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 3301
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657354944.819, "dur": 17.889,
    "args": {
      "External id": 2486503,"Record function id": 0, "Ev Idx": 3302
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657354951.369, "dur": 10.457,
    "args": {
      "External id": 2486504,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3303
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657354955.558, "dur": 5.416,
    "args": {
      "External id": 2486505,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3304
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657354957.119, "dur": 3.742,
    "args": {
      "External id": 2486506,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3305
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657354967.301, "dur": 7.137,
    "args": {
      "External id": 2486507,"Record function id": 0, "Ev Idx": 3306
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657354969.123, "dur": 4.880,
    "args": {
      "External id": 2486508,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3307
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657354969.697, "dur": 3.821,
    "args": {
      "External id": 2486509,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3308
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657354970.278, "dur": 3.134,
    "args": {
      "External id": 2486510,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3309
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657354977.986, "dur": 46.963,
    "args": {
      "External id": 2486511,"Record function id": 0, "Ev Idx": 3310
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657354979.658, "dur": 44.215,
    "args": {
      "External id": 2486512,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3311
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657354980.489, "dur": 42.234,
    "args": {
      "External id": 2486513,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3312
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657355020.676, "dur": 1.740,
    "args": {
      "External id": 2486514,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3313
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657355030.678, "dur": 5.919,
    "args": {
      "External id": 2486515,"Record function id": 0, "Ev Idx": 3314
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657355032.393, "dur": 3.727,
    "args": {
      "External id": 2486516,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3315
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657355033.532, "dur": 1.910,
    "args": {
      "External id": 2486517,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3316
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657355034.347, "dur": 0.969,
    "args": {
      "External id": 2486518,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3317
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657355040.107, "dur": 4.221,
    "args": {
      "External id": 2486519,"Record function id": 0, "Ev Idx": 3318
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657355041.458, "dur": 2.476,
    "args": {
      "External id": 2486520,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3319
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657355041.958, "dur": 1.264,
    "args": {
      "External id": 2486521,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3320
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657355042.242, "dur": 0.917,
    "args": {
      "External id": 2486522,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3321
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657355047.832, "dur": 3.782,
    "args": {
      "External id": 2486523,"Record function id": 0, "Ev Idx": 3322
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657355049.108, "dur": 2.107,
    "args": {
      "External id": 2486524,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3323
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657355049.696, "dur": 1.068,
    "args": {
      "External id": 2486525,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3324
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657355049.975, "dur": 0.671,
    "args": {
      "External id": 2486526,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3325
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657355054.906, "dur": 4.102,
    "args": {
      "External id": 2486527,"Record function id": 0, "Ev Idx": 3326
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657355056.056, "dur": 2.577,
    "args": {
      "External id": 2486528,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3327
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657355056.634, "dur": 1.449,
    "args": {
      "External id": 2486529,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3328
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657355057.189, "dur": 0.778,
    "args": {
      "External id": 2486530,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3329
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657355062.412, "dur": 4.031,
    "args": {
      "External id": 2486531,"Record function id": 0, "Ev Idx": 3330
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657355063.764, "dur": 2.303,
    "args": {
      "External id": 2486532,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3331
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657355064.207, "dur": 1.325,
    "args": {
      "External id": 2486533,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3332
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657355064.720, "dur": 0.754,
    "args": {
      "External id": 2486534,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3333
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657355069.584, "dur": 5.794,
    "args": {
      "External id": 2486535,"Record function id": 0, "Ev Idx": 3334
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657355070.890, "dur": 4.106,
    "args": {
      "External id": 2486536,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3335
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657355071.587, "dur": 2.713,
    "args": {
      "External id": 2486537,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3336
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657355071.982, "dur": 2.225,
    "args": {
      "External id": 2486538,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3337
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657355079.389, "dur": 37210.635,
    "args": {
      "External id": 2486539,"Record function id": 0, "Sequence number": 24740453, "Fwd thread id": 1, "Ev Idx": 3338
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657355081.019, "dur": 37200.015,
    "args": {
      "External id": 2486540,"Sequence number": 24740453, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 3339
    }
  },
  {
    "ph": "f", "id": 195, "pid": 1336754, "tid": 1381179, "ts": 1514657355081.019,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.16)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657355111.478, "dur": 38.098,
    "args": {
      "External id": 2486541,"Record function id": 0, "Ev Idx": 3340
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.16)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657355157.167, "dur": 66.826,
    "args": {
      "External id": 2486542,"Record function id": 0, "Ev Idx": 3341
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.16)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657355231.292, "dur": 37041.276,
    "args": {
      "External id": 2486543,"Record function id": 0, "Ev Idx": 3342
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657355321.815, "dur": 7.551,
    "args": {
      "External id": 2486544,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3343
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657355338.936, "dur": 7.857,
    "args": {
      "External id": 2486545,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3344
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514657355361.644, "dur": 35987.234,
    "args": {
      "External id": 2486546,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3345
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514657355375.526, "dur": 35964.207,
    "args": {
      "External id": 2486547,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3346
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657355439.229, "dur": 15.138,
    "args": {
      "External id": 2486548,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3347
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514657355460.889, "dur": 35838.748,
    "args": {
      "External id": 2486549,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 3348
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657355463.601, "dur": 35835.345,
    "args": {
      "External id": 2486550,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 3349
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657355467.547, "dur": 4.823,
    "args": {
      "External id": 2486551,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3350
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657355474.061, "dur": 35821.375,
    "args": {
      "External id": 2486552,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 3351
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657391444.387, "dur": 9.277,
    "args": {
      "External id": 2486553,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 3352
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657391447.403, "dur": 5.929,
    "args": {
      "External id": 2486554,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3353
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657391482.493, "dur": 403.305,
    "args": {
      "External id": 2486555,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 3354
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657391512.920, "dur": 367.947,
    "args": {
      "External id": 2486556,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 3355, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514657391523.900, "dur": 350.762,
    "args": {
      "External id": 2486557,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 3356
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657391917.038, "dur": 2.873,
    "args": {
      "External id": 2486558,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 3357, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657392039.191, "dur": 6.962,
    "args": {
      "External id": 2486559,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3358
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657392094.656, "dur": 3.295,
    "args": {
      "External id": 2486560,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3359
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657392115.729, "dur": 1.015,
    "args": {
      "External id": 2486561,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3360
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657392131.456, "dur": 0.905,
    "args": {
      "External id": 2486562,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3361
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657392145.688, "dur": 0.798,
    "args": {
      "External id": 2486563,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3362
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657392158.843, "dur": 2.350,
    "args": {
      "External id": 2486564,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3363
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657392174.374, "dur": 1.309,
    "args": {
      "External id": 2486565,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3364
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657392189.519, "dur": 2.220,
    "args": {
      "External id": 2486566,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3365
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657392204.794, "dur": 0.991,
    "args": {
      "External id": 2486567,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3366
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657392306.273, "dur": 2935.410,
    "args": {
      "External id": 2486568,"Record function id": 0, "Ev Idx": 3367
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.15)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657392327.783, "dur": 1110.122,
    "args": {
      "External id": 2486569,"Record function id": 0, "Ev Idx": 3368
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.15)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657392345.399, "dur": 358.405,
    "args": {
      "External id": 2486570,"Record function id": 0, "Ev Idx": 3369
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657392428.432, "dur": 5.967,
    "args": {
      "External id": 2486571,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3370
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657392438.064, "dur": 1.393,
    "args": {
      "External id": 2486572,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3371
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657392441.513, "dur": 0.979,
    "args": {
      "External id": 2486573,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3372
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657392444.487, "dur": 1.249,
    "args": {
      "External id": 2486574,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3373
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657392447.249, "dur": 1.140,
    "args": {
      "External id": 2486575,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3374
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657392450.284, "dur": 1.135,
    "args": {
      "External id": 2486576,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3375
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657392453.049, "dur": 1.744,
    "args": {
      "External id": 2486577,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3376
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657392456.378, "dur": 1.201,
    "args": {
      "External id": 2486578,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3377
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657392459.201, "dur": 3.313,
    "args": {
      "External id": 2486579,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3378
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657392464.325, "dur": 1.030,
    "args": {
      "External id": 2486580,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3379
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657392482.771, "dur": 184.588,
    "args": {
      "External id": 2486581,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3380
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657392499.608, "dur": 162.498,
    "args": {
      "External id": 2486582,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3381
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657392519.974, "dur": 28.907,
    "args": {
      "External id": 2486583,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3382
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514657392553.358, "dur": 77.879,
    "args": {
      "External id": 2486584,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 3383
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657392556.214, "dur": 74.670,
    "args": {
      "External id": 2486585,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 3384
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657392564.302, "dur": 5.845,
    "args": {
      "External id": 2486586,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3385
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657392571.995, "dur": 58.398,
    "args": {
      "External id": 2486587,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 3386
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.14", "pid": 1336754, "tid": 1381179,
    "ts": 1514657392800.023, "dur": 629.872,
    "args": {
      "External id": 2486588,"Record function id": 0, "Ev Idx": 3387
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.14)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657392816.630, "dur": 599.434,
    "args": {
      "External id": 2486589,"Record function id": 0, "Ev Idx": 3388
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657392887.365, "dur": 8.687,
    "args": {
      "External id": 2486590,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3389
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1381179,
    "ts": 1514657392912.474, "dur": 32.051,
    "args": {
      "External id": 2486591,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 3390
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657392917.904, "dur": 1.554,
    "args": {
      "External id": 2486592,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3391
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657392921.832, "dur": 0.467,
    "args": {
      "External id": 2486593,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3392
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657392924.363, "dur": 0.451,
    "args": {
      "External id": 2486594,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3393
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657392926.647, "dur": 2.149,
    "args": {
      "External id": 2486595,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3394
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657392930.661, "dur": 0.346,
    "args": {
      "External id": 2486596,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3395
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657392932.757, "dur": 0.699,
    "args": {
      "External id": 2486597,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3396
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657392935.148, "dur": 0.568,
    "args": {
      "External id": 2486598,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3397
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657392937.550, "dur": 0.471,
    "args": {
      "External id": 2486599,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3398
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657392939.935, "dur": 0.430,
    "args": {
      "External id": 2486600,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3399
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657392958.438, "dur": 73.935,
    "args": {
      "External id": 2486601,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 3400
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1381179,
    "ts": 1514657393069.569, "dur": 119.481,
    "args": {
      "External id": 2486602,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "1", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 3401
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657393084.524, "dur": 4.204,
    "args": {
      "External id": 2486603,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3402
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1381179,
    "ts": 1514657393094.315, "dur": 11.108,
    "args": {
      "External id": 2486604,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 3403
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1381179,
    "ts": 1514657393098.637, "dur": 6.360,
    "args": {
      "External id": 2486605,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "12846080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 3404
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657393102.762, "dur": 0.671,
    "args": {
      "External id": 2486606,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 3405
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1381179,
    "ts": 1514657393113.473, "dur": 31.501,
    "args": {
      "External id": 2486607,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 3406
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657393116.193, "dur": 2.438,
    "args": {
      "External id": 2486608,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3407
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657393120.435, "dur": 0.435,
    "args": {
      "External id": 2486609,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6423296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3408
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657393122.526, "dur": 0.571,
    "args": {
      "External id": 2486610,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6947584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3409
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657393124.694, "dur": 0.379,
    "args": {
      "External id": 2486611,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7471872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3410
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657393127.164, "dur": 0.610,
    "args": {
      "External id": 2486612,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7996160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3411
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657393129.359, "dur": 0.325,
    "args": {
      "External id": 2486613,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8520448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3412
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657393131.606, "dur": 0.645,
    "args": {
      "External id": 2486614,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "8520704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3413
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657393133.973, "dur": 0.466,
    "args": {
      "External id": 2486615,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "9962496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3414
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657393136.312, "dur": 2.385,
    "args": {
      "External id": 2486616,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11404288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3415
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657393156.465, "dur": 24.556,
    "args": {
      "External id": 2486617,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 3416
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657393239.287, "dur": 115.455,
    "args": {
      "External id": 2486618,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 3417
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657393264.665, "dur": 86.515,
    "args": {
      "External id": 2486619,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 3418, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514657393274.895, "dur": 72.160,
    "args": {
      "External id": 2486620,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 3419
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657393367.113, "dur": 1.831,
    "args": {
      "External id": 2486621,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 3420, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657393445.638, "dur": 1773.879,
    "args": {
      "External id": 2486622,"Sequence number": 24740452, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 3421
    }
  },
  {
    "ph": "f", "id": 196, "pid": 1336754, "tid": 1381179, "ts": 1514657393445.638,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657393572.611, "dur": 107.765,
    "args": {
      "External id": 2486623,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 3422
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514657393719.111, "dur": 43.255,
    "args": {
      "External id": 2486624,"kernel_hash": "cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/fu/cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 3423
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657393783.473, "dur": 52.706,
    "args": {
      "External id": 2486625,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 3424
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657393846.367, "dur": 34.421,
    "args": {
      "External id": 2486626,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 3425
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657393888.585, "dur": 46.440,
    "args": {
      "External id": 2486627,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 3426
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657393943.840, "dur": 28.705,
    "args": {
      "External id": 2486628,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 3427
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657394020.140, "dur": 50.816,
    "args": {
      "External id": 2486629,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 3428
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657394096.282, "dur": 26.093,
    "args": {
      "External id": 2486630,"kernel_hash": "cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/wm/cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 3429
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514657394142.933, "dur": 32.050,
    "args": {
      "External id": 2486631,"kernel_hash": "cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/br/cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3430
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657394199.562, "dur": 20.759,
    "args": {
      "External id": 2486632,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 3431
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514657394233.199, "dur": 16.141,
    "args": {
      "External id": 2486633,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 3432
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657394262.086, "dur": 31.218,
    "args": {
      "External id": 2486634,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3433
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657394296.459, "dur": 34.262,
    "args": {
      "External id": 2486635,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3434
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657394376.605, "dur": 195.104,
    "args": {
      "External id": 2486636,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 3435
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657394451.288, "dur": 5.800,
    "args": {
      "External id": 2486637,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3436
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657394459.360, "dur": 2.240,
    "args": {
      "External id": 2486638,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3437
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657394612.560, "dur": 32.043,
    "args": {
      "External id": 2486639,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3438
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657394656.400, "dur": 17.088,
    "args": {
      "External id": 2486640,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3439
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657394683.616, "dur": 63.450,
    "args": {
      "External id": 2486641,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3440
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657394765.500, "dur": 46.937,
    "args": {
      "External id": 2486642,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3441
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657394820.827, "dur": 23.640,
    "args": {
      "External id": 2486643,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3442
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657394850.186, "dur": 35.762,
    "args": {
      "External id": 2486644,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3443
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657394892.581, "dur": 21.740,
    "args": {
      "External id": 2486645,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3444
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657394928.128, "dur": 30.687,
    "args": {
      "External id": 2486646,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3445
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336754, "tid": 1381179,
    "ts": 1514657394977.793, "dur": 62.380,
    "args": {
      "External id": 2486647,"kernel_hash": "c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/6w/c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 3446
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657395063.922, "dur": 25.703,
    "args": {
      "External id": 2486648,"kernel_hash": "c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/4k/c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3447
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657395105.312, "dur": 19.672,
    "args": {
      "External id": 2486649,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 3448
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514657395141.135, "dur": 15.973,
    "args": {
      "External id": 2486650,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 3449
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336754, "tid": 1381179,
    "ts": 1514657395173.366, "dur": 17.166,
    "args": {
      "External id": 2486651,"kernel_hash": "cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/eb/cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 3450
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657395265.279, "dur": 17.253,
    "args": {
      "External id": 2486652,"Record function id": 0, "Ev Idx": 3451
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657395268.647, "dur": 13.007,
    "args": {
      "External id": 2486653,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3452
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657395273.007, "dur": 7.784,
    "args": {
      "External id": 2486654,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3453
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657395274.752, "dur": 5.894,
    "args": {
      "External id": 2486655,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3454
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657395286.720, "dur": 5.979,
    "args": {
      "External id": 2486656,"Record function id": 0, "Ev Idx": 3455
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657395288.007, "dur": 4.228,
    "args": {
      "External id": 2486657,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3456
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657395288.989, "dur": 2.483,
    "args": {
      "External id": 2486658,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3457
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657395290.350, "dur": 0.990,
    "args": {
      "External id": 2486659,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3458
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657395295.871, "dur": 4.396,
    "args": {
      "External id": 2486660,"Record function id": 0, "Ev Idx": 3459
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657395297.193, "dur": 2.683,
    "args": {
      "External id": 2486661,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3460
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657395297.760, "dur": 1.671,
    "args": {
      "External id": 2486662,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3461
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657395298.358, "dur": 1.000,
    "args": {
      "External id": 2486663,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3462
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657395303.409, "dur": 4.057,
    "args": {
      "External id": 2486664,"Record function id": 0, "Ev Idx": 3463
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657395304.819, "dur": 2.253,
    "args": {
      "External id": 2486665,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3464
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657395305.410, "dur": 0.996,
    "args": {
      "External id": 2486666,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3465
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657395305.700, "dur": 0.630,
    "args": {
      "External id": 2486667,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3466
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657395310.568, "dur": 3.898,
    "args": {
      "External id": 2486668,"Record function id": 0, "Ev Idx": 3467
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657395311.966, "dur": 2.072,
    "args": {
      "External id": 2486669,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3468
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657395312.557, "dur": 1.049,
    "args": {
      "External id": 2486670,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3469
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657395312.812, "dur": 0.724,
    "args": {
      "External id": 2486671,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3470
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657395317.946, "dur": 3.936,
    "args": {
      "External id": 2486672,"Record function id": 0, "Ev Idx": 3471
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657395319.064, "dur": 2.410,
    "args": {
      "External id": 2486673,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3472
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657395319.591, "dur": 1.420,
    "args": {
      "External id": 2486674,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3473
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657395320.134, "dur": 0.743,
    "args": {
      "External id": 2486675,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3474
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657395325.080, "dur": 3.374,
    "args": {
      "External id": 2486676,"Record function id": 0, "Ev Idx": 3475
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657395326.054, "dur": 1.955,
    "args": {
      "External id": 2486677,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3476
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657395326.578, "dur": 0.966,
    "args": {
      "External id": 2486678,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3477
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657395326.876, "dur": 0.569,
    "args": {
      "External id": 2486679,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3478
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657395331.611, "dur": 6.068,
    "args": {
      "External id": 2486680,"Record function id": 0, "Ev Idx": 3479
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657395332.680, "dur": 4.550,
    "args": {
      "External id": 2486681,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3480
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657395333.349, "dur": 3.188,
    "args": {
      "External id": 2486682,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3481
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657395333.731, "dur": 2.719,
    "args": {
      "External id": 2486683,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3482
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657395341.077, "dur": 3.867,
    "args": {
      "External id": 2486684,"Record function id": 0, "Ev Idx": 3483
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657395342.210, "dur": 2.309,
    "args": {
      "External id": 2486685,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3484
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657395342.656, "dur": 1.424,
    "args": {
      "External id": 2486686,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3485
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657395343.073, "dur": 0.881,
    "args": {
      "External id": 2486687,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3486
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657395348.516, "dur": 35471.787,
    "args": {
      "External id": 2486688,"Record function id": 0, "Sequence number": 24740451, "Fwd thread id": 1, "Ev Idx": 3487
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657395349.905, "dur": 35462.100,
    "args": {
      "External id": 2486689,"Sequence number": 24740451, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 3488
    }
  },
  {
    "ph": "f", "id": 197, "pid": 1336754, "tid": 1381179, "ts": 1514657395349.905,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.15)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657395379.308, "dur": 39.665,
    "args": {
      "External id": 2486690,"Record function id": 0, "Ev Idx": 3489
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.15)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657395427.554, "dur": 67.139,
    "args": {
      "External id": 2486691,"Record function id": 0, "Ev Idx": 3490
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.15)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657395500.446, "dur": 35303.390,
    "args": {
      "External id": 2486692,"Record function id": 0, "Ev Idx": 3491
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657395611.372, "dur": 7.292,
    "args": {
      "External id": 2486693,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3492
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657395628.998, "dur": 4.819,
    "args": {
      "External id": 2486694,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3493
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514657395650.103, "dur": 34328.662,
    "args": {
      "External id": 2486695,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3494
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514657395663.654, "dur": 34306.302,
    "args": {
      "External id": 2486696,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3495
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657395722.418, "dur": 14.256,
    "args": {
      "External id": 2486697,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3496
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514657395743.256, "dur": 34184.369,
    "args": {
      "External id": 2486698,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 3497
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657395745.711, "dur": 34181.238,
    "args": {
      "External id": 2486699,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 3498
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657395749.840, "dur": 4.982,
    "args": {
      "External id": 2486700,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3499
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657395756.797, "dur": 34166.404,
    "args": {
      "External id": 2486701,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 3500
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657430097.265, "dur": 10.347,
    "args": {
      "External id": 2486702,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 3501
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657430101.083, "dur": 5.879,
    "args": {
      "External id": 2486703,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3502
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657430137.731, "dur": 348.466,
    "args": {
      "External id": 2486704,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 3503
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657430164.611, "dur": 316.924,
    "args": {
      "External id": 2486705,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 3504, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514657430176.586, "dur": 299.285,
    "args": {
      "External id": 2486706,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 3505
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657430504.126, "dur": 2.490,
    "args": {
      "External id": 2486707,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 3506, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657430582.325, "dur": 6.876,
    "args": {
      "External id": 2486708,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3507
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657430637.440, "dur": 1.517,
    "args": {
      "External id": 2486709,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3508
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657430656.201, "dur": 1.277,
    "args": {
      "External id": 2486710,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3509
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657430670.928, "dur": 1.062,
    "args": {
      "External id": 2486711,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3510
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657430683.905, "dur": 1.205,
    "args": {
      "External id": 2486712,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3511
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657430699.231, "dur": 0.988,
    "args": {
      "External id": 2486713,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3512
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657430712.985, "dur": 0.933,
    "args": {
      "External id": 2486714,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3513
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657430726.746, "dur": 1.929,
    "args": {
      "External id": 2486715,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3514
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657430740.114, "dur": 0.802,
    "args": {
      "External id": 2486716,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3515
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657430836.195, "dur": 2930.377,
    "args": {
      "External id": 2486717,"Record function id": 0, "Ev Idx": 3516
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.14)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657430858.983, "dur": 1084.821,
    "args": {
      "External id": 2486718,"Record function id": 0, "Ev Idx": 3517
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.14)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657430874.937, "dur": 371.020,
    "args": {
      "External id": 2486719,"Record function id": 0, "Ev Idx": 3518
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657430957.981, "dur": 4.158,
    "args": {
      "External id": 2486720,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3519
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657430965.158, "dur": 1.543,
    "args": {
      "External id": 2486721,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3520
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657430968.644, "dur": 1.306,
    "args": {
      "External id": 2486722,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3521
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657430971.549, "dur": 1.052,
    "args": {
      "External id": 2486723,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3522
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657430974.208, "dur": 1.007,
    "args": {
      "External id": 2486724,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3523
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657430977.161, "dur": 1.233,
    "args": {
      "External id": 2486725,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3524
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657430979.988, "dur": 36.079,
    "args": {
      "External id": 2486726,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3525
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657431020.550, "dur": 3.785,
    "args": {
      "External id": 2486727,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3526
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657431026.007, "dur": 1.199,
    "args": {
      "External id": 2486728,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3527
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657431029.008, "dur": 1.058,
    "args": {
      "External id": 2486729,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3528
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657431050.346, "dur": 162.145,
    "args": {
      "External id": 2486730,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3529
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657431067.381, "dur": 140.527,
    "args": {
      "External id": 2486731,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3530
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657431092.075, "dur": 12.324,
    "args": {
      "External id": 2486732,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3531
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514657431107.523, "dur": 71.053,
    "args": {
      "External id": 2486733,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 3532
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657431110.503, "dur": 67.701,
    "args": {
      "External id": 2486734,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 3533
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657431114.236, "dur": 6.394,
    "args": {
      "External id": 2486735,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3534
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657431122.745, "dur": 54.801,
    "args": {
      "External id": 2486736,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 3535
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.13", "pid": 1336754, "tid": 1381179,
    "ts": 1514657431342.121, "dur": 594.099,
    "args": {
      "External id": 2486737,"Record function id": 0, "Ev Idx": 3536
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.13)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657431359.744, "dur": 563.589,
    "args": {
      "External id": 2486738,"Record function id": 0, "Ev Idx": 3537
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657431427.574, "dur": 5.116,
    "args": {
      "External id": 2486739,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3538
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1381179,
    "ts": 1514657431448.221, "dur": 30.030,
    "args": {
      "External id": 2486740,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 3539
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657431453.572, "dur": 1.791,
    "args": {
      "External id": 2486741,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3540
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657431457.615, "dur": 0.464,
    "args": {
      "External id": 2486742,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3541
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657431459.870, "dur": 2.366,
    "args": {
      "External id": 2486743,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3542
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657431464.063, "dur": 0.426,
    "args": {
      "External id": 2486744,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3543
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657431465.798, "dur": 0.453,
    "args": {
      "External id": 2486745,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3544
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657431467.760, "dur": 0.411,
    "args": {
      "External id": 2486746,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3545
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657431469.843, "dur": 0.397,
    "args": {
      "External id": 2486747,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3546
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657431471.854, "dur": 0.549,
    "args": {
      "External id": 2486748,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3547
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657431474.005, "dur": 0.405,
    "args": {
      "External id": 2486749,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3548
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657431489.494, "dur": 33.377,
    "args": {
      "External id": 2486750,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 3549
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1381179,
    "ts": 1514657431574.742, "dur": 122.565,
    "args": {
      "External id": 2486751,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "1", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 3550
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657431586.519, "dur": 4.613,
    "args": {
      "External id": 2486752,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3551
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1381179,
    "ts": 1514657431602.365, "dur": 13.637,
    "args": {
      "External id": 2486753,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 3552
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1381179,
    "ts": 1514657431606.808, "dur": 8.741,
    "args": {
      "External id": 2486754,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "12846080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 3553
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657431610.920, "dur": 2.785,
    "args": {
      "External id": 2486755,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 3554
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1381179,
    "ts": 1514657431626.248, "dur": 25.641,
    "args": {
      "External id": 2486756,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 3555
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657431629.083, "dur": 0.623,
    "args": {
      "External id": 2486757,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3556
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657431631.572, "dur": 0.430,
    "args": {
      "External id": 2486758,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6423296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3557
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657431633.670, "dur": 0.369,
    "args": {
      "External id": 2486759,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6947584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3558
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657431635.928, "dur": 0.430,
    "args": {
      "External id": 2486760,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7471872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3559
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657431637.757, "dur": 0.589,
    "args": {
      "External id": 2486761,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7996160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3560
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657431640.055, "dur": 0.370,
    "args": {
      "External id": 2486762,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8520448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3561
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657431641.938, "dur": 0.451,
    "args": {
      "External id": 2486763,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "8520704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3562
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657431644.133, "dur": 1.844,
    "args": {
      "External id": 2486764,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "9962496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3563
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657431647.766, "dur": 0.424,
    "args": {
      "External id": 2486765,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11404288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3564
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657431666.106, "dur": 23.155,
    "args": {
      "External id": 2486766,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 3565
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657431746.610, "dur": 113.652,
    "args": {
      "External id": 2486767,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 3566
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657431771.764, "dur": 85.064,
    "args": {
      "External id": 2486768,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 3567, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514657431781.340, "dur": 71.451,
    "args": {
      "External id": 2486769,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 3568
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657431873.585, "dur": 2.007,
    "args": {
      "External id": 2486770,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 3569, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657431951.502, "dur": 1792.861,
    "args": {
      "External id": 2486771,"Sequence number": 24740450, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 3570
    }
  },
  {
    "ph": "f", "id": 198, "pid": 1336754, "tid": 1381179, "ts": 1514657431951.502,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657432106.320, "dur": 115.668,
    "args": {
      "External id": 2486772,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 3571
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514657432264.461, "dur": 46.741,
    "args": {
      "External id": 2486773,"kernel_hash": "cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/fu/cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 3572
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657432330.439, "dur": 55.100,
    "args": {
      "External id": 2486774,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 3573
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657432395.762, "dur": 37.270,
    "args": {
      "External id": 2486775,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 3574
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657432440.869, "dur": 47.552,
    "args": {
      "External id": 2486776,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 3575
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657432496.139, "dur": 28.251,
    "args": {
      "External id": 2486777,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 3576
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657432555.329, "dur": 47.251,
    "args": {
      "External id": 2486778,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 3577
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657432626.545, "dur": 24.031,
    "args": {
      "External id": 2486779,"kernel_hash": "cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/wm/cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 3578
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514657432668.766, "dur": 31.331,
    "args": {
      "External id": 2486780,"kernel_hash": "cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/br/cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3579
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657432720.135, "dur": 19.846,
    "args": {
      "External id": 2486781,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 3580
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514657432755.979, "dur": 16.249,
    "args": {
      "External id": 2486782,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 3581
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657432782.294, "dur": 30.346,
    "args": {
      "External id": 2486783,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3582
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657432816.101, "dur": 33.486,
    "args": {
      "External id": 2486784,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3583
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657432897.456, "dur": 215.654,
    "args": {
      "External id": 2486785,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 3584
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657432972.868, "dur": 5.957,
    "args": {
      "External id": 2486786,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3585
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657433014.959, "dur": 4.150,
    "args": {
      "External id": 2486787,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3586
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657433149.089, "dur": 25.568,
    "args": {
      "External id": 2486788,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3587
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657433186.167, "dur": 17.053,
    "args": {
      "External id": 2486789,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3588
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657433218.852, "dur": 59.012,
    "args": {
      "External id": 2486790,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3589
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657433290.877, "dur": 46.091,
    "args": {
      "External id": 2486791,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3590
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657433345.104, "dur": 22.662,
    "args": {
      "External id": 2486792,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3591
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657433373.493, "dur": 34.959,
    "args": {
      "External id": 2486793,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3592
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657433417.343, "dur": 21.641,
    "args": {
      "External id": 2486794,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3593
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657433447.045, "dur": 34.981,
    "args": {
      "External id": 2486795,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3594
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336754, "tid": 1381179,
    "ts": 1514657433500.490, "dur": 41.201,
    "args": {
      "External id": 2486796,"kernel_hash": "c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/6w/c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 3595
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657433571.592, "dur": 26.342,
    "args": {
      "External id": 2486797,"kernel_hash": "c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/4k/c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3596
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657433616.085, "dur": 19.606,
    "args": {
      "External id": 2486798,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 3597
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514657433656.091, "dur": 17.580,
    "args": {
      "External id": 2486799,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 3598
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336754, "tid": 1381179,
    "ts": 1514657433688.243, "dur": 25.787,
    "args": {
      "External id": 2486800,"kernel_hash": "cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/eb/cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 3599
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657433789.184, "dur": 14.882,
    "args": {
      "External id": 2486801,"Record function id": 0, "Ev Idx": 3600
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657433792.556, "dur": 10.598,
    "args": {
      "External id": 2486802,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3601
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657433796.285, "dur": 5.887,
    "args": {
      "External id": 2486803,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3602
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657433797.878, "dur": 4.167,
    "args": {
      "External id": 2486804,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3603
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657433808.065, "dur": 5.360,
    "args": {
      "External id": 2486805,"Record function id": 0, "Ev Idx": 3604
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657433809.536, "dur": 3.474,
    "args": {
      "External id": 2486806,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3605
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657433810.597, "dur": 1.947,
    "args": {
      "External id": 2486807,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3606
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657433811.384, "dur": 1.053,
    "args": {
      "External id": 2486808,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3607
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657433816.650, "dur": 8.976,
    "args": {
      "External id": 2486809,"Record function id": 0, "Ev Idx": 3608
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657433818.350, "dur": 6.878,
    "args": {
      "External id": 2486810,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3609
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657433822.549, "dur": 2.120,
    "args": {
      "External id": 2486811,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3610
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657433823.377, "dur": 1.205,
    "args": {
      "External id": 2486812,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3611
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657433828.804, "dur": 4.419,
    "args": {
      "External id": 2486813,"Record function id": 0, "Ev Idx": 3612
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657433830.016, "dur": 2.796,
    "args": {
      "External id": 2486814,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3613
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657433830.931, "dur": 1.461,
    "args": {
      "External id": 2486815,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3614
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657433831.625, "dur": 0.681,
    "args": {
      "External id": 2486816,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3615
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657433836.409, "dur": 4.543,
    "args": {
      "External id": 2486817,"Record function id": 0, "Ev Idx": 3616
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657433837.833, "dur": 2.720,
    "args": {
      "External id": 2486818,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3617
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657433838.319, "dur": 1.641,
    "args": {
      "External id": 2486819,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3618
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657433839.376, "dur": 0.518,
    "args": {
      "External id": 2486820,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3619
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657433844.091, "dur": 3.854,
    "args": {
      "External id": 2486821,"Record function id": 0, "Ev Idx": 3620
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657433845.139, "dur": 2.399,
    "args": {
      "External id": 2486822,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3621
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657433845.824, "dur": 1.285,
    "args": {
      "External id": 2486823,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3622
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657433846.310, "dur": 0.698,
    "args": {
      "External id": 2486824,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3623
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657433851.230, "dur": 5.651,
    "args": {
      "External id": 2486825,"Record function id": 0, "Ev Idx": 3624
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657433852.427, "dur": 4.066,
    "args": {
      "External id": 2486826,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3625
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657433852.957, "dur": 3.100,
    "args": {
      "External id": 2486827,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3626
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657433853.249, "dur": 2.716,
    "args": {
      "External id": 2486828,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3627
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657433860.078, "dur": 3.960,
    "args": {
      "External id": 2486829,"Record function id": 0, "Ev Idx": 3628
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657433861.292, "dur": 2.337,
    "args": {
      "External id": 2486830,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3629
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657433862.101, "dur": 1.116,
    "args": {
      "External id": 2486831,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3630
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657433862.369, "dur": 0.770,
    "args": {
      "External id": 2486832,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3631
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657433867.105, "dur": 3.855,
    "args": {
      "External id": 2486833,"Record function id": 0, "Ev Idx": 3632
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657433868.223, "dur": 2.333,
    "args": {
      "External id": 2486834,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3633
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657433868.885, "dur": 1.244,
    "args": {
      "External id": 2486835,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3634
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657433869.366, "dur": 0.671,
    "args": {
      "External id": 2486836,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3635
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657433874.897, "dur": 35404.066,
    "args": {
      "External id": 2486837,"Record function id": 0, "Sequence number": 24740449, "Fwd thread id": 1, "Ev Idx": 3636
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657433876.687, "dur": 35392.628,
    "args": {
      "External id": 2486838,"Sequence number": 24740449, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 3637
    }
  },
  {
    "ph": "f", "id": 199, "pid": 1336754, "tid": 1381179, "ts": 1514657433876.687,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.14)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657433910.738, "dur": 39.548,
    "args": {
      "External id": 2486839,"Record function id": 0, "Ev Idx": 3638
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.14)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657433957.541, "dur": 105.779,
    "args": {
      "External id": 2486840,"Record function id": 0, "Ev Idx": 3639
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.14)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657434075.118, "dur": 35186.172,
    "args": {
      "External id": 2486841,"Record function id": 0, "Ev Idx": 3640
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657434167.987, "dur": 7.065,
    "args": {
      "External id": 2486842,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3641
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657434185.363, "dur": 8.257,
    "args": {
      "External id": 2486843,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3642
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514657434209.017, "dur": 34221.929,
    "args": {
      "External id": 2486844,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3643
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514657434222.763, "dur": 34199.253,
    "args": {
      "External id": 2486845,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3644
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657434265.501, "dur": 14.806,
    "args": {
      "External id": 2486846,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3645
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514657434286.931, "dur": 34092.332,
    "args": {
      "External id": 2486847,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 3646
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657434289.520, "dur": 34089.061,
    "args": {
      "External id": 2486848,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 3647
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657434293.485, "dur": 5.420,
    "args": {
      "External id": 2486849,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3648
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657434300.642, "dur": 34074.645,
    "args": {
      "External id": 2486850,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 3649
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657468517.907, "dur": 9.401,
    "args": {
      "External id": 2486851,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 3650
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657468521.276, "dur": 5.727,
    "args": {
      "External id": 2486852,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3651
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657468568.467, "dur": 361.021,
    "args": {
      "External id": 2486853,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 3652
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657468594.987, "dur": 329.576,
    "args": {
      "External id": 2486854,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 3653, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514657468606.482, "dur": 312.116,
    "args": {
      "External id": 2486855,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 3654
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657468949.386, "dur": 2.488,
    "args": {
      "External id": 2486856,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 3655, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657469043.534, "dur": 8.745,
    "args": {
      "External id": 2486857,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3656
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657469098.780, "dur": 1.406,
    "args": {
      "External id": 2486858,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3657
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657469117.075, "dur": 0.942,
    "args": {
      "External id": 2486859,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3658
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657469131.428, "dur": 1.454,
    "args": {
      "External id": 2486860,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3659
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657469145.336, "dur": 2.797,
    "args": {
      "External id": 2486861,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3660
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657469158.559, "dur": 1.017,
    "args": {
      "External id": 2486862,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3661
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657469171.968, "dur": 1.058,
    "args": {
      "External id": 2486863,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3662
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657469185.483, "dur": 2.334,
    "args": {
      "External id": 2486864,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3663
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657469198.585, "dur": 3.038,
    "args": {
      "External id": 2486865,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3664
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657469295.181, "dur": 2857.657,
    "args": {
      "External id": 2486866,"Record function id": 0, "Ev Idx": 3665
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.13)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657469313.685, "dur": 1051.542,
    "args": {
      "External id": 2486867,"Record function id": 0, "Ev Idx": 3666
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.13)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657469329.342, "dur": 340.457,
    "args": {
      "External id": 2486868,"Record function id": 0, "Ev Idx": 3667
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657469410.515, "dur": 3.840,
    "args": {
      "External id": 2486869,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3668
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657469417.416, "dur": 1.235,
    "args": {
      "External id": 2486870,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3669
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657469420.533, "dur": 1.011,
    "args": {
      "External id": 2486871,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3670
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657469423.477, "dur": 1.240,
    "args": {
      "External id": 2486872,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3671
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657469426.506, "dur": 1.090,
    "args": {
      "External id": 2486873,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3672
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657469429.092, "dur": 1.017,
    "args": {
      "External id": 2486874,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3673
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657469431.405, "dur": 3.386,
    "args": {
      "External id": 2486875,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3674
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657469436.755, "dur": 1.326,
    "args": {
      "External id": 2486876,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3675
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657469439.624, "dur": 0.822,
    "args": {
      "External id": 2486877,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3676
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657469442.182, "dur": 0.705,
    "args": {
      "External id": 2486878,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3677
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657469462.895, "dur": 172.782,
    "args": {
      "External id": 2486879,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3678
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657469478.633, "dur": 151.806,
    "args": {
      "External id": 2486880,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3679
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657469501.812, "dur": 12.541,
    "args": {
      "External id": 2486881,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3680
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514657469517.545, "dur": 83.340,
    "args": {
      "External id": 2486882,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 3681
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657469520.169, "dur": 80.334,
    "args": {
      "External id": 2486883,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 3682
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657469524.192, "dur": 19.763,
    "args": {
      "External id": 2486884,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3683
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657469546.456, "dur": 53.473,
    "args": {
      "External id": 2486885,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 3684
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.12", "pid": 1336754, "tid": 1381179,
    "ts": 1514657469762.230, "dur": 595.646,
    "args": {
      "External id": 2486886,"Record function id": 0, "Ev Idx": 3685
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.12)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657469779.164, "dur": 566.039,
    "args": {
      "External id": 2486887,"Record function id": 0, "Ev Idx": 3686
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657469849.339, "dur": 5.580,
    "args": {
      "External id": 2486888,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3687
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1381179,
    "ts": 1514657469874.256, "dur": 30.709,
    "args": {
      "External id": 2486889,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 3688
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657469879.362, "dur": 1.570,
    "args": {
      "External id": 2486890,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3689
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657469883.017, "dur": 2.680,
    "args": {
      "External id": 2486891,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3690
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657469887.837, "dur": 0.562,
    "args": {
      "External id": 2486892,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3691
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657469890.168, "dur": 0.338,
    "args": {
      "External id": 2486893,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3692
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657469892.424, "dur": 0.362,
    "args": {
      "External id": 2486894,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3693
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657469894.513, "dur": 0.559,
    "args": {
      "External id": 2486895,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3694
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657469896.826, "dur": 0.388,
    "args": {
      "External id": 2486896,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3695
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657469898.650, "dur": 0.376,
    "args": {
      "External id": 2486897,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3696
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657469900.460, "dur": 0.367,
    "args": {
      "External id": 2486898,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3697
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657469916.145, "dur": 32.013,
    "args": {
      "External id": 2486899,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 3698
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1381179,
    "ts": 1514657469980.108, "dur": 145.164,
    "args": {
      "External id": 2486900,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "1", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 3699
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657470027.317, "dur": 6.551,
    "args": {
      "External id": 2486901,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3700
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1381179,
    "ts": 1514657470039.536, "dur": 10.815,
    "args": {
      "External id": 2486902,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 3701
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1381179,
    "ts": 1514657470043.778, "dur": 6.160,
    "args": {
      "External id": 2486903,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "12846080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 3702
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657470047.828, "dur": 0.625,
    "args": {
      "External id": 2486904,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 3703
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1381179,
    "ts": 1514657470057.729, "dur": 24.553,
    "args": {
      "External id": 2486905,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 3704
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657470059.903, "dur": 0.384,
    "args": {
      "External id": 2486906,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3705
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657470062.273, "dur": 0.787,
    "args": {
      "External id": 2486907,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6423296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3706
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657470064.870, "dur": 0.781,
    "args": {
      "External id": 2486908,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6947584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3707
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657470067.062, "dur": 0.358,
    "args": {
      "External id": 2486909,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7471872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3708
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657470068.762, "dur": 0.436,
    "args": {
      "External id": 2486910,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7996160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3709
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657470070.787, "dur": 0.373,
    "args": {
      "External id": 2486911,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8520448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3710
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657470072.756, "dur": 2.151,
    "args": {
      "External id": 2486912,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "8520704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3711
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657470076.267, "dur": 0.372,
    "args": {
      "External id": 2486913,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "9962496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3712
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657470078.168, "dur": 0.337,
    "args": {
      "External id": 2486914,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11404288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3713
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657470093.596, "dur": 23.582,
    "args": {
      "External id": 2486915,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 3714
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657470171.157, "dur": 110.971,
    "args": {
      "External id": 2486916,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 3715
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657470192.068, "dur": 86.738,
    "args": {
      "External id": 2486917,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 3716, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514657470202.268, "dur": 72.440,
    "args": {
      "External id": 2486918,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 3717
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657470295.160, "dur": 1.860,
    "args": {
      "External id": 2486919,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 3718, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657470373.887, "dur": 1757.011,
    "args": {
      "External id": 2486920,"Sequence number": 24740448, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 3719
    }
  },
  {
    "ph": "f", "id": 200, "pid": 1336754, "tid": 1381179, "ts": 1514657470373.887,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657470485.234, "dur": 129.795,
    "args": {
      "External id": 2486921,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 3720
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514657470655.848, "dur": 46.182,
    "args": {
      "External id": 2486922,"kernel_hash": "cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/fu/cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 3721
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657470721.461, "dur": 57.236,
    "args": {
      "External id": 2486923,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 3722
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657470787.869, "dur": 38.121,
    "args": {
      "External id": 2486924,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 3723
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657470833.551, "dur": 48.268,
    "args": {
      "External id": 2486925,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 3724
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657470892.888, "dur": 32.543,
    "args": {
      "External id": 2486926,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 3725
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657470933.175, "dur": 43.486,
    "args": {
      "External id": 2486927,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 3726
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657471047.176, "dur": 29.650,
    "args": {
      "External id": 2486928,"kernel_hash": "cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/wm/cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 3727
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514657471095.269, "dur": 34.615,
    "args": {
      "External id": 2486929,"kernel_hash": "cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/br/cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3728
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657471149.053, "dur": 20.464,
    "args": {
      "External id": 2486930,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 3729
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514657471183.192, "dur": 16.233,
    "args": {
      "External id": 2486931,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 3730
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657471210.624, "dur": 35.756,
    "args": {
      "External id": 2486932,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3731
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657471249.889, "dur": 35.636,
    "args": {
      "External id": 2486933,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3732
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657471313.245, "dur": 172.622,
    "args": {
      "External id": 2486934,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 3733
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657471387.713, "dur": 5.754,
    "args": {
      "External id": 2486935,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3734
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657471395.372, "dur": 4.377,
    "args": {
      "External id": 2486936,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3735
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657471518.104, "dur": 42.170,
    "args": {
      "External id": 2486937,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3736
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657471588.373, "dur": 17.744,
    "args": {
      "External id": 2486938,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3737
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657471617.140, "dur": 42.207,
    "args": {
      "External id": 2486939,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3738
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657471666.843, "dur": 54.860,
    "args": {
      "External id": 2486940,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3739
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657471735.136, "dur": 26.185,
    "args": {
      "External id": 2486941,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3740
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657471770.348, "dur": 34.885,
    "args": {
      "External id": 2486942,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3741
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657471812.087, "dur": 20.282,
    "args": {
      "External id": 2486943,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3742
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657471840.325, "dur": 33.849,
    "args": {
      "External id": 2486944,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3743
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336754, "tid": 1381179,
    "ts": 1514657471892.055, "dur": 23.123,
    "args": {
      "External id": 2486945,"kernel_hash": "c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/6w/c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 3744
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657471932.402, "dur": 24.241,
    "args": {
      "External id": 2486946,"kernel_hash": "c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/4k/c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3745
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657471971.342, "dur": 52.499,
    "args": {
      "External id": 2486947,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 3746
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514657472046.636, "dur": 18.480,
    "args": {
      "External id": 2486948,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 3747
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336754, "tid": 1381179,
    "ts": 1514657472081.485, "dur": 17.798,
    "args": {
      "External id": 2486949,"kernel_hash": "cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/eb/cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 3748
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657472176.080, "dur": 19.362,
    "args": {
      "External id": 2486950,"Record function id": 0, "Ev Idx": 3749
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657472183.015, "dur": 11.613,
    "args": {
      "External id": 2486951,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3750
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657472187.542, "dur": 6.202,
    "args": {
      "External id": 2486952,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3751
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657472189.602, "dur": 3.987,
    "args": {
      "External id": 2486953,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3752
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657472199.293, "dur": 5.731,
    "args": {
      "External id": 2486954,"Record function id": 0, "Ev Idx": 3753
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657472200.828, "dur": 3.764,
    "args": {
      "External id": 2486955,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3754
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657472202.040, "dur": 2.049,
    "args": {
      "External id": 2486956,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3755
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657472202.872, "dur": 1.119,
    "args": {
      "External id": 2486957,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3756
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657472208.270, "dur": 4.956,
    "args": {
      "External id": 2486958,"Record function id": 0, "Ev Idx": 3757
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657472209.378, "dur": 3.389,
    "args": {
      "External id": 2486959,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3758
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657472210.424, "dur": 1.784,
    "args": {
      "External id": 2486960,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3759
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657472210.779, "dur": 1.352,
    "args": {
      "External id": 2486961,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3760
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657472216.374, "dur": 6.211,
    "args": {
      "External id": 2486962,"Record function id": 0, "Ev Idx": 3761
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657472217.759, "dur": 4.387,
    "args": {
      "External id": 2486963,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3762
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657472218.381, "dur": 3.348,
    "args": {
      "External id": 2486964,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3763
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657472218.724, "dur": 2.937,
    "args": {
      "External id": 2486965,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3764
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657472225.726, "dur": 4.031,
    "args": {
      "External id": 2486966,"Record function id": 0, "Ev Idx": 3765
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657472227.033, "dur": 2.304,
    "args": {
      "External id": 2486967,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3766
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657472227.519, "dur": 1.389,
    "args": {
      "External id": 2486968,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3767
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657472227.951, "dur": 0.886,
    "args": {
      "External id": 2486969,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3768
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657472232.858, "dur": 7.337,
    "args": {
      "External id": 2486970,"Record function id": 0, "Ev Idx": 3769
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657472233.934, "dur": 5.828,
    "args": {
      "External id": 2486971,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3770
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657472234.633, "dur": 1.239,
    "args": {
      "External id": 2486972,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3771
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657472235.122, "dur": 0.641,
    "args": {
      "External id": 2486973,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3772
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657472243.435, "dur": 3.833,
    "args": {
      "External id": 2486974,"Record function id": 0, "Ev Idx": 3773
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657472244.524, "dur": 2.328,
    "args": {
      "External id": 2486975,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3774
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657472245.020, "dur": 1.397,
    "args": {
      "External id": 2486976,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3775
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657472245.426, "dur": 0.887,
    "args": {
      "External id": 2486977,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3776
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657472250.385, "dur": 3.974,
    "args": {
      "External id": 2486978,"Record function id": 0, "Ev Idx": 3777
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657472251.468, "dur": 2.436,
    "args": {
      "External id": 2486979,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3778
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657472252.074, "dur": 1.405,
    "args": {
      "External id": 2486980,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3779
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657472252.716, "dur": 0.689,
    "args": {
      "External id": 2486981,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3780
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657472257.482, "dur": 4.188,
    "args": {
      "External id": 2486982,"Record function id": 0, "Ev Idx": 3781
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657472258.604, "dur": 2.678,
    "args": {
      "External id": 2486983,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3782
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657472259.083, "dur": 1.727,
    "args": {
      "External id": 2486984,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3783
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657472259.588, "dur": 1.127,
    "args": {
      "External id": 2486985,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3784
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657472265.597, "dur": 38103.350,
    "args": {
      "External id": 2486986,"Record function id": 0, "Sequence number": 24740447, "Fwd thread id": 1, "Ev Idx": 3785
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657472267.266, "dur": 38093.578,
    "args": {
      "External id": 2486987,"Sequence number": 24740447, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 3786
    }
  },
  {
    "ph": "f", "id": 201, "pid": 1336754, "tid": 1381179, "ts": 1514657472267.266,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.13)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657472299.255, "dur": 37.664,
    "args": {
      "External id": 2486988,"Record function id": 0, "Ev Idx": 3787
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.13)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657472344.159, "dur": 73.977,
    "args": {
      "External id": 2486989,"Record function id": 0, "Ev Idx": 3788
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.13)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657472423.786, "dur": 37929.266,
    "args": {
      "External id": 2486990,"Record function id": 0, "Ev Idx": 3789
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657472514.029, "dur": 6.117,
    "args": {
      "External id": 2486991,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3790
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657472545.832, "dur": 5.125,
    "args": {
      "External id": 2486992,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3791
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514657472570.572, "dur": 36874.460,
    "args": {
      "External id": 2486993,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3792
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514657472584.582, "dur": 36851.671,
    "args": {
      "External id": 2486994,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3793
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657472628.141, "dur": 17.378,
    "args": {
      "External id": 2486995,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3794
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514657472652.224, "dur": 36743.358,
    "args": {
      "External id": 2486996,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 3795
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657472658.128, "dur": 36736.758,
    "args": {
      "External id": 2486997,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 3796
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657472662.115, "dur": 5.846,
    "args": {
      "External id": 2486998,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3797
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657472669.761, "dur": 36721.309,
    "args": {
      "External id": 2486999,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 3798
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657509553.100, "dur": 10.224,
    "args": {
      "External id": 2487000,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 3799
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657509556.385, "dur": 6.331,
    "args": {
      "External id": 2487001,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3800
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657509593.012, "dur": 435.358,
    "args": {
      "External id": 2487002,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 3801
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657509618.266, "dur": 404.177,
    "args": {
      "External id": 2487003,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 3802, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514657509629.923, "dur": 385.528,
    "args": {
      "External id": 2487004,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 3803
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657510054.844, "dur": 2.925,
    "args": {
      "External id": 2487005,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 3804, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657510125.279, "dur": 7.002,
    "args": {
      "External id": 2487006,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3805
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657510179.250, "dur": 1.768,
    "args": {
      "External id": 2487007,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3806
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657510199.147, "dur": 0.955,
    "args": {
      "External id": 2487008,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3807
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657510220.199, "dur": 1.302,
    "args": {
      "External id": 2487009,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3808
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657510234.691, "dur": 0.837,
    "args": {
      "External id": 2487010,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3809
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657510247.588, "dur": 1.148,
    "args": {
      "External id": 2487011,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3810
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657510260.832, "dur": 0.935,
    "args": {
      "External id": 2487012,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3811
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657510277.563, "dur": 1.947,
    "args": {
      "External id": 2487013,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3812
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657510291.874, "dur": 1.035,
    "args": {
      "External id": 2487014,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3813
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657510383.809, "dur": 2880.477,
    "args": {
      "External id": 2487015,"Record function id": 0, "Ev Idx": 3814
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.12)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657510403.724, "dur": 1073.356,
    "args": {
      "External id": 2487016,"Record function id": 0, "Ev Idx": 3815
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.12)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657510419.334, "dur": 351.980,
    "args": {
      "External id": 2487017,"Record function id": 0, "Ev Idx": 3816
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657510504.347, "dur": 4.274,
    "args": {
      "External id": 2487018,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3817
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657510512.161, "dur": 0.972,
    "args": {
      "External id": 2487019,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3818
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657510514.750, "dur": 0.993,
    "args": {
      "External id": 2487020,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3819
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657510517.562, "dur": 2.544,
    "args": {
      "External id": 2487021,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3820
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657510521.589, "dur": 1.239,
    "args": {
      "External id": 2487022,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3821
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657510524.410, "dur": 1.106,
    "args": {
      "External id": 2487023,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3822
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657510527.113, "dur": 18.034,
    "args": {
      "External id": 2487024,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3823
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657510548.869, "dur": 1.185,
    "args": {
      "External id": 2487025,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3824
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657510551.720, "dur": 0.826,
    "args": {
      "External id": 2487026,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3825
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657510554.088, "dur": 1.132,
    "args": {
      "External id": 2487027,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3826
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657510574.204, "dur": 163.223,
    "args": {
      "External id": 2487028,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3827
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657510590.766, "dur": 141.714,
    "args": {
      "External id": 2487029,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3828
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657510614.095, "dur": 13.559,
    "args": {
      "External id": 2487030,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3829
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514657510631.174, "dur": 71.737,
    "args": {
      "External id": 2487031,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 3830
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657510633.881, "dur": 68.602,
    "args": {
      "External id": 2487032,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 3831
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657510637.584, "dur": 8.428,
    "args": {
      "External id": 2487033,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3832
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657510647.938, "dur": 54.126,
    "args": {
      "External id": 2487034,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 3833
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.11", "pid": 1336754, "tid": 1381179,
    "ts": 1514657510864.675, "dur": 604.152,
    "args": {
      "External id": 2487035,"Record function id": 0, "Ev Idx": 3834
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.11)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657510883.135, "dur": 573.114,
    "args": {
      "External id": 2487036,"Record function id": 0, "Ev Idx": 3835
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657510945.062, "dur": 4.791,
    "args": {
      "External id": 2487037,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3836
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1381179,
    "ts": 1514657510965.704, "dur": 71.539,
    "args": {
      "External id": 2487038,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 3837
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657510970.670, "dur": 1.598,
    "args": {
      "External id": 2487039,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3838
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657510974.613, "dur": 0.658,
    "args": {
      "External id": 2487040,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3839
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657510976.939, "dur": 0.368,
    "args": {
      "External id": 2487041,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3840
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657510978.705, "dur": 0.375,
    "args": {
      "External id": 2487042,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3841
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657510980.461, "dur": 39.229,
    "args": {
      "External id": 2487043,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3842
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657511023.281, "dur": 0.867,
    "args": {
      "External id": 2487044,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3843
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657511025.863, "dur": 2.457,
    "args": {
      "External id": 2487045,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3844
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657511030.225, "dur": 0.545,
    "args": {
      "External id": 2487046,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3845
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657511032.356, "dur": 0.402,
    "args": {
      "External id": 2487047,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3846
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657511050.254, "dur": 37.504,
    "args": {
      "External id": 2487048,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 3847
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1381179,
    "ts": 1514657511127.393, "dur": 106.871,
    "args": {
      "External id": 2487049,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "1", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 3848
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657511138.557, "dur": 4.048,
    "args": {
      "External id": 2487050,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3849
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1381179,
    "ts": 1514657511147.765, "dur": 10.871,
    "args": {
      "External id": 2487051,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 3850
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1381179,
    "ts": 1514657511152.272, "dur": 5.926,
    "args": {
      "External id": 2487052,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "12846080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 3851
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657511156.062, "dur": 0.649,
    "args": {
      "External id": 2487053,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 3852
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1381179,
    "ts": 1514657511169.479, "dur": 24.401,
    "args": {
      "External id": 2487054,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 3853
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657511171.924, "dur": 0.420,
    "args": {
      "External id": 2487055,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3854
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657511174.205, "dur": 0.347,
    "args": {
      "External id": 2487056,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6423296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3855
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657511175.966, "dur": 0.723,
    "args": {
      "External id": 2487057,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6947584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3856
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657511178.137, "dur": 2.576,
    "args": {
      "External id": 2487058,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7471872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3857
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657511182.266, "dur": 0.361,
    "args": {
      "External id": 2487059,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7996160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3858
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657511184.043, "dur": 0.409,
    "args": {
      "External id": 2487060,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8520448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3859
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657511185.948, "dur": 0.400,
    "args": {
      "External id": 2487061,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "8520704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3860
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657511187.921, "dur": 0.483,
    "args": {
      "External id": 2487062,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "9962496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3861
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657511190.087, "dur": 0.381,
    "args": {
      "External id": 2487063,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11404288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3862
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657511206.026, "dur": 20.519,
    "args": {
      "External id": 2487064,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 3863
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657511281.152, "dur": 110.593,
    "args": {
      "External id": 2487065,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 3864
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657511302.716, "dur": 85.813,
    "args": {
      "External id": 2487066,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 3865, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514657511312.538, "dur": 71.949,
    "args": {
      "External id": 2487067,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 3866
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657511404.279, "dur": 1.687,
    "args": {
      "External id": 2487068,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 3867, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657511485.066, "dur": 1757.116,
    "args": {
      "External id": 2487069,"Sequence number": 24740446, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 3868
    }
  },
  {
    "ph": "f", "id": 202, "pid": 1336754, "tid": 1381179, "ts": 1514657511485.066,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657511611.627, "dur": 110.686,
    "args": {
      "External id": 2487070,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 3869
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514657511766.419, "dur": 43.604,
    "args": {
      "External id": 2487071,"kernel_hash": "cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/fu/cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 3870
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657511827.960, "dur": 52.001,
    "args": {
      "External id": 2487072,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 3871
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657511889.686, "dur": 34.353,
    "args": {
      "External id": 2487073,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 3872
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657511933.096, "dur": 46.748,
    "args": {
      "External id": 2487074,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 3873
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657512029.441, "dur": 39.188,
    "args": {
      "External id": 2487075,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 3874
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657512076.992, "dur": 48.792,
    "args": {
      "External id": 2487076,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 3875
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657512150.812, "dur": 25.316,
    "args": {
      "External id": 2487077,"kernel_hash": "cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/wm/cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 3876
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514657512199.830, "dur": 30.793,
    "args": {
      "External id": 2487078,"kernel_hash": "cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/br/cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3877
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657512253.474, "dur": 20.987,
    "args": {
      "External id": 2487079,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 3878
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514657512288.344, "dur": 18.641,
    "args": {
      "External id": 2487080,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 3879
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657512316.406, "dur": 30.245,
    "args": {
      "External id": 2487081,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3880
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657512350.310, "dur": 37.551,
    "args": {
      "External id": 2487082,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3881
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657512420.666, "dur": 193.576,
    "args": {
      "External id": 2487083,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 3882
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657512498.376, "dur": 5.802,
    "args": {
      "External id": 2487084,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3883
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657512506.315, "dur": 1.948,
    "args": {
      "External id": 2487085,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3884
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657512650.281, "dur": 25.414,
    "args": {
      "External id": 2487086,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3885
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657512687.928, "dur": 16.032,
    "args": {
      "External id": 2487087,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3886
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657512713.716, "dur": 48.102,
    "args": {
      "External id": 2487088,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3887
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657512791.482, "dur": 49.257,
    "args": {
      "External id": 2487089,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3888
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657512851.618, "dur": 23.888,
    "args": {
      "External id": 2487090,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3889
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657512884.883, "dur": 31.103,
    "args": {
      "External id": 2487091,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3890
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657512922.830, "dur": 24.455,
    "args": {
      "External id": 2487092,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3891
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657512955.754, "dur": 64.984,
    "args": {
      "External id": 2487093,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3892
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336754, "tid": 1381179,
    "ts": 1514657513044.549, "dur": 26.420,
    "args": {
      "External id": 2487094,"kernel_hash": "c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/6w/c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 3893
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657513088.967, "dur": 25.718,
    "args": {
      "External id": 2487095,"kernel_hash": "c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/4k/c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3894
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657513129.766, "dur": 18.729,
    "args": {
      "External id": 2487096,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 3895
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514657513165.099, "dur": 17.043,
    "args": {
      "External id": 2487097,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 3896
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336754, "tid": 1381179,
    "ts": 1514657513196.115, "dur": 18.405,
    "args": {
      "External id": 2487098,"kernel_hash": "cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/eb/cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 3897
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657513288.828, "dur": 15.688,
    "args": {
      "External id": 2487099,"Record function id": 0, "Ev Idx": 3898
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657513292.619, "dur": 10.970,
    "args": {
      "External id": 2487100,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3899
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657513297.073, "dur": 5.502,
    "args": {
      "External id": 2487101,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3900
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657513298.707, "dur": 3.755,
    "args": {
      "External id": 2487102,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3901
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657513308.390, "dur": 6.180,
    "args": {
      "External id": 2487103,"Record function id": 0, "Ev Idx": 3902
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657513309.756, "dur": 4.388,
    "args": {
      "External id": 2487104,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3903
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657513310.899, "dur": 2.656,
    "args": {
      "External id": 2487105,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3904
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657513312.203, "dur": 1.232,
    "args": {
      "External id": 2487106,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3905
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657513317.936, "dur": 7.349,
    "args": {
      "External id": 2487107,"Record function id": 0, "Ev Idx": 3906
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657513319.408, "dur": 5.474,
    "args": {
      "External id": 2487108,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3907
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657513320.389, "dur": 3.953,
    "args": {
      "External id": 2487109,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3908
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657513320.909, "dur": 3.345,
    "args": {
      "External id": 2487110,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3909
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657513328.625, "dur": 4.720,
    "args": {
      "External id": 2487111,"Record function id": 0, "Ev Idx": 3910
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657513329.853, "dur": 3.056,
    "args": {
      "External id": 2487112,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3911
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657513330.881, "dur": 1.617,
    "args": {
      "External id": 2487113,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3912
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657513331.419, "dur": 1.007,
    "args": {
      "External id": 2487114,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3913
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657513336.428, "dur": 7.003,
    "args": {
      "External id": 2487115,"Record function id": 0, "Ev Idx": 3914
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657513337.810, "dur": 5.231,
    "args": {
      "External id": 2487116,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3915
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657513338.448, "dur": 4.151,
    "args": {
      "External id": 2487117,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3916
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657513338.771, "dur": 3.762,
    "args": {
      "External id": 2487118,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3917
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657513346.488, "dur": 4.608,
    "args": {
      "External id": 2487119,"Record function id": 0, "Ev Idx": 3918
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657513347.717, "dur": 2.959,
    "args": {
      "External id": 2487120,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3919
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657513348.436, "dur": 1.558,
    "args": {
      "External id": 2487121,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3920
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657513349.020, "dur": 0.874,
    "args": {
      "External id": 2487122,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3921
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657513354.435, "dur": 4.053,
    "args": {
      "External id": 2487123,"Record function id": 0, "Ev Idx": 3922
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657513355.684, "dur": 2.412,
    "args": {
      "External id": 2487124,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3923
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657513356.347, "dur": 1.096,
    "args": {
      "External id": 2487125,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3924
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657513356.714, "dur": 0.644,
    "args": {
      "External id": 2487126,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3925
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657513361.574, "dur": 4.318,
    "args": {
      "External id": 2487127,"Record function id": 0, "Ev Idx": 3926
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657513362.846, "dur": 2.644,
    "args": {
      "External id": 2487128,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3927
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657513363.476, "dur": 1.197,
    "args": {
      "External id": 2487129,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3928
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657513363.736, "dur": 0.871,
    "args": {
      "External id": 2487130,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3929
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657513368.951, "dur": 29.470,
    "args": {
      "External id": 2487131,"Record function id": 0, "Ev Idx": 3930
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657513370.345, "dur": 27.630,
    "args": {
      "External id": 2487132,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3931
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657513370.999, "dur": 26.265,
    "args": {
      "External id": 2487133,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3932
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657513396.542, "dur": 0.598,
    "args": {
      "External id": 2487134,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3933
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657513402.591, "dur": 35428.313,
    "args": {
      "External id": 2487135,"Record function id": 0, "Sequence number": 24740445, "Fwd thread id": 1, "Ev Idx": 3934
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657513404.100, "dur": 35417.884,
    "args": {
      "External id": 2487136,"Sequence number": 24740445, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 3935
    }
  },
  {
    "ph": "f", "id": 203, "pid": 1336754, "tid": 1381179, "ts": 1514657513404.100,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.12)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657513435.832, "dur": 35.449,
    "args": {
      "External id": 2487137,"Record function id": 0, "Ev Idx": 3936
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.12)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657513478.615, "dur": 86.854,
    "args": {
      "External id": 2487138,"Record function id": 0, "Ev Idx": 3937
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.12)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657513573.070, "dur": 35241.139,
    "args": {
      "External id": 2487139,"Record function id": 0, "Ev Idx": 3938
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657513661.545, "dur": 10.864,
    "args": {
      "External id": 2487140,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3939
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657513682.718, "dur": 6.907,
    "args": {
      "External id": 2487141,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3940
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514657513705.514, "dur": 34270.358,
    "args": {
      "External id": 2487142,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3941
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514657513719.104, "dur": 34247.393,
    "args": {
      "External id": 2487143,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3942
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657513762.854, "dur": 14.794,
    "args": {
      "External id": 2487144,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3943
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514657513784.347, "dur": 34139.982,
    "args": {
      "External id": 2487145,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 3944
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657513786.929, "dur": 34136.621,
    "args": {
      "External id": 2487146,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 3945
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657513791.083, "dur": 5.438,
    "args": {
      "External id": 2487147,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3946
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657513798.622, "dur": 34121.629,
    "args": {
      "External id": 2487148,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 3947
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657548088.728, "dur": 9.496,
    "args": {
      "External id": 2487149,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 3948
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657548091.769, "dur": 5.795,
    "args": {
      "External id": 2487150,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3949
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657548128.418, "dur": 362.965,
    "args": {
      "External id": 2487151,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 3950
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657548154.540, "dur": 330.906,
    "args": {
      "External id": 2487152,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 3951, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514657548166.417, "dur": 312.724,
    "args": {
      "External id": 2487153,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 3952
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657548511.696, "dur": 2.660,
    "args": {
      "External id": 2487154,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 3953, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657548589.545, "dur": 7.208,
    "args": {
      "External id": 2487155,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3954
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657548648.554, "dur": 1.623,
    "args": {
      "External id": 2487156,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3955
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657548667.734, "dur": 3.240,
    "args": {
      "External id": 2487157,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3956
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657548684.573, "dur": 0.841,
    "args": {
      "External id": 2487158,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3957
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657548697.591, "dur": 0.946,
    "args": {
      "External id": 2487159,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3958
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657548710.986, "dur": 0.970,
    "args": {
      "External id": 2487160,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3959
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657548724.051, "dur": 3.069,
    "args": {
      "External id": 2487161,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3960
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657548739.086, "dur": 2.264,
    "args": {
      "External id": 2487162,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3961
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657548752.348, "dur": 0.969,
    "args": {
      "External id": 2487163,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3962
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657548845.852, "dur": 2832.000,
    "args": {
      "External id": 2487164,"Record function id": 0, "Ev Idx": 3963
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.11)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657548864.639, "dur": 1043.751,
    "args": {
      "External id": 2487165,"Record function id": 0, "Ev Idx": 3964
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.11)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657548880.364, "dur": 378.056,
    "args": {
      "External id": 2487166,"Record function id": 0, "Ev Idx": 3965
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657548964.256, "dur": 4.275,
    "args": {
      "External id": 2487167,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3966
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657548971.577, "dur": 1.275,
    "args": {
      "External id": 2487168,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3967
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657548974.726, "dur": 3.757,
    "args": {
      "External id": 2487169,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3968
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657548979.917, "dur": 1.165,
    "args": {
      "External id": 2487170,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3969
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657549018.432, "dur": 1.798,
    "args": {
      "External id": 2487171,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3970
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657549022.257, "dur": 1.008,
    "args": {
      "External id": 2487172,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3971
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657549025.027, "dur": 1.692,
    "args": {
      "External id": 2487173,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3972
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657549028.257, "dur": 1.235,
    "args": {
      "External id": 2487174,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3973
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657549031.066, "dur": 1.500,
    "args": {
      "External id": 2487175,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3974
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657549034.166, "dur": 1.112,
    "args": {
      "External id": 2487176,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3975
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657549054.440, "dur": 167.761,
    "args": {
      "External id": 2487177,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3976
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657549074.320, "dur": 143.151,
    "args": {
      "External id": 2487178,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3977
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657549093.891, "dur": 14.349,
    "args": {
      "External id": 2487179,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3978
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514657549111.652, "dur": 76.457,
    "args": {
      "External id": 2487180,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 3979
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657549114.330, "dur": 73.397,
    "args": {
      "External id": 2487181,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 3980
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657549121.554, "dur": 6.268,
    "args": {
      "External id": 2487182,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3981
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657549129.713, "dur": 57.576,
    "args": {
      "External id": 2487183,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 3982
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.10", "pid": 1336754, "tid": 1381179,
    "ts": 1514657549350.653, "dur": 549.594,
    "args": {
      "External id": 2487184,"Record function id": 0, "Ev Idx": 3983
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.10)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657549368.504, "dur": 519.364,
    "args": {
      "External id": 2487185,"Record function id": 0, "Ev Idx": 3984
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657549430.056, "dur": 5.307,
    "args": {
      "External id": 2487186,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3985
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1381179,
    "ts": 1514657549450.982, "dur": 28.497,
    "args": {
      "External id": 2487187,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 3986
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657549455.902, "dur": 1.362,
    "args": {
      "External id": 2487188,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3987
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657549459.331, "dur": 0.402,
    "args": {
      "External id": 2487189,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3988
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657549461.161, "dur": 0.860,
    "args": {
      "External id": 2487190,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3989
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657549463.627, "dur": 0.457,
    "args": {
      "External id": 2487191,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3990
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657549465.529, "dur": 0.440,
    "args": {
      "External id": 2487192,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3991
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657549467.342, "dur": 2.199,
    "args": {
      "External id": 2487193,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3992
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657549471.049, "dur": 0.286,
    "args": {
      "External id": 2487194,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3993
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657549472.930, "dur": 0.588,
    "args": {
      "External id": 2487195,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3994
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657549475.299, "dur": 0.627,
    "args": {
      "External id": 2487196,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3995
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657549488.929, "dur": 31.771,
    "args": {
      "External id": 2487197,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 3996
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1381179,
    "ts": 1514657549569.192, "dur": 105.928,
    "args": {
      "External id": 2487198,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "1", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 3997
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657549579.631, "dur": 4.323,
    "args": {
      "External id": 2487199,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3998
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1381179,
    "ts": 1514657549589.116, "dur": 10.768,
    "args": {
      "External id": 2487200,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 3999
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1381179,
    "ts": 1514657549593.179, "dur": 6.305,
    "args": {
      "External id": 2487201,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "12846080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 4000
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657549597.094, "dur": 0.715,
    "args": {
      "External id": 2487202,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 4001
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1381179,
    "ts": 1514657549607.276, "dur": 26.273,
    "args": {
      "External id": 2487203,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 4002
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657549609.871, "dur": 0.922,
    "args": {
      "External id": 2487204,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4003
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657549612.666, "dur": 0.426,
    "args": {
      "External id": 2487205,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6423296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4004
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657549614.637, "dur": 2.879,
    "args": {
      "External id": 2487206,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6947584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4005
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657549619.076, "dur": 0.351,
    "args": {
      "External id": 2487207,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7471872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4006
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657549621.307, "dur": 0.370,
    "args": {
      "External id": 2487208,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7996160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4007
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657549623.233, "dur": 0.523,
    "args": {
      "External id": 2487209,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8520448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4008
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657549625.486, "dur": 0.781,
    "args": {
      "External id": 2487210,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "8520704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4009
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657549627.727, "dur": 0.446,
    "args": {
      "External id": 2487211,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "9962496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4010
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657549629.978, "dur": 0.516,
    "args": {
      "External id": 2487212,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11404288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4011
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657549644.193, "dur": 23.651,
    "args": {
      "External id": 2487213,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 4012
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657549721.890, "dur": 104.825,
    "args": {
      "External id": 2487214,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 4013
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657549742.193, "dur": 81.190,
    "args": {
      "External id": 2487215,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4014, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514657549751.818, "dur": 67.817,
    "args": {
      "External id": 2487216,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 4015
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657549839.287, "dur": 1.870,
    "args": {
      "External id": 2487217,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4016, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657549915.593, "dur": 1740.104,
    "args": {
      "External id": 2487218,"Sequence number": 24740444, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4017
    }
  },
  {
    "ph": "f", "id": 204, "pid": 1336754, "tid": 1381179, "ts": 1514657549915.593,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657550068.220, "dur": 108.670,
    "args": {
      "External id": 2487219,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 4018
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514657550216.020, "dur": 43.521,
    "args": {
      "External id": 2487220,"kernel_hash": "cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/fu/cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 4019
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657550277.379, "dur": 55.867,
    "args": {
      "External id": 2487221,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 4020
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657550345.692, "dur": 39.077,
    "args": {
      "External id": 2487222,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4021
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657550391.628, "dur": 49.304,
    "args": {
      "External id": 2487223,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4022
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657550449.360, "dur": 32.801,
    "args": {
      "External id": 2487224,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4023
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657550489.737, "dur": 61.108,
    "args": {
      "External id": 2487225,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4024
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657550579.267, "dur": 31.286,
    "args": {
      "External id": 2487226,"kernel_hash": "cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/wm/cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 4025
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514657550629.958, "dur": 35.039,
    "args": {
      "External id": 2487227,"kernel_hash": "cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/br/cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4026
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657550684.122, "dur": 23.799,
    "args": {
      "External id": 2487228,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4027
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514657550721.214, "dur": 16.622,
    "args": {
      "External id": 2487229,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4028
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657550747.237, "dur": 36.961,
    "args": {
      "External id": 2487230,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4029
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657550787.735, "dur": 35.348,
    "args": {
      "External id": 2487231,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4030
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657550852.447, "dur": 216.733,
    "args": {
      "External id": 2487232,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 4031
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657550929.674, "dur": 5.757,
    "args": {
      "External id": 2487233,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4032
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657550937.494, "dur": 2.196,
    "args": {
      "External id": 2487234,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4033
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657551108.483, "dur": 28.082,
    "args": {
      "External id": 2487235,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4034
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657551147.081, "dur": 16.596,
    "args": {
      "External id": 2487236,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4035
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657551173.238, "dur": 43.359,
    "args": {
      "External id": 2487237,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4036
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657551223.772, "dur": 53.346,
    "args": {
      "External id": 2487238,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4037
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657551292.797, "dur": 25.482,
    "args": {
      "External id": 2487239,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4038
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657551324.066, "dur": 33.886,
    "args": {
      "External id": 2487240,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4039
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657551367.606, "dur": 20.994,
    "args": {
      "External id": 2487241,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4040
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657551396.216, "dur": 35.238,
    "args": {
      "External id": 2487242,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4041
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336754, "tid": 1381179,
    "ts": 1514657551451.097, "dur": 23.300,
    "args": {
      "External id": 2487243,"kernel_hash": "c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/6w/c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 4042
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657551491.253, "dur": 23.951,
    "args": {
      "External id": 2487244,"kernel_hash": "c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/4k/c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4043
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657551544.113, "dur": 21.346,
    "args": {
      "External id": 2487245,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4044
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514657551582.571, "dur": 15.993,
    "args": {
      "External id": 2487246,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4045
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336754, "tid": 1381179,
    "ts": 1514657551611.178, "dur": 17.668,
    "args": {
      "External id": 2487247,"kernel_hash": "cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/eb/cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 4046
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657551699.386, "dur": 16.152,
    "args": {
      "External id": 2487248,"Record function id": 0, "Ev Idx": 4047
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657551703.192, "dur": 11.286,
    "args": {
      "External id": 2487249,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4048
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657551707.639, "dur": 5.946,
    "args": {
      "External id": 2487250,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4049
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657551709.535, "dur": 3.877,
    "args": {
      "External id": 2487251,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4050
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657551719.310, "dur": 7.665,
    "args": {
      "External id": 2487252,"Record function id": 0, "Ev Idx": 4051
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657551721.042, "dur": 5.464,
    "args": {
      "External id": 2487253,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4052
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657551721.739, "dur": 4.025,
    "args": {
      "External id": 2487254,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4053
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657551722.716, "dur": 2.921,
    "args": {
      "External id": 2487255,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4054
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657551730.219, "dur": 4.707,
    "args": {
      "External id": 2487256,"Record function id": 0, "Ev Idx": 4055
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657551731.679, "dur": 2.839,
    "args": {
      "External id": 2487257,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4056
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657551732.551, "dur": 1.480,
    "args": {
      "External id": 2487258,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4057
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657551733.105, "dur": 0.852,
    "args": {
      "External id": 2487259,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4058
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657551738.207, "dur": 4.054,
    "args": {
      "External id": 2487260,"Record function id": 0, "Ev Idx": 4059
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657551739.684, "dur": 2.138,
    "args": {
      "External id": 2487261,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4060
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657551740.204, "dur": 1.174,
    "args": {
      "External id": 2487262,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4061
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657551740.524, "dur": 0.791,
    "args": {
      "External id": 2487263,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4062
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657551745.331, "dur": 4.393,
    "args": {
      "External id": 2487264,"Record function id": 0, "Ev Idx": 4063
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657551746.824, "dur": 2.508,
    "args": {
      "External id": 2487265,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4064
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657551747.868, "dur": 1.023,
    "args": {
      "External id": 2487266,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4065
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657551748.133, "dur": 0.684,
    "args": {
      "External id": 2487267,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4066
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657551752.800, "dur": 4.128,
    "args": {
      "External id": 2487268,"Record function id": 0, "Ev Idx": 4067
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657551754.250, "dur": 2.260,
    "args": {
      "External id": 2487269,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4068
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657551754.696, "dur": 1.340,
    "args": {
      "External id": 2487270,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4069
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657551755.368, "dur": 0.572,
    "args": {
      "External id": 2487271,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4070
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657551763.060, "dur": 4.544,
    "args": {
      "External id": 2487272,"Record function id": 0, "Ev Idx": 4071
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657551764.209, "dur": 2.978,
    "args": {
      "External id": 2487273,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4072
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657551764.893, "dur": 1.629,
    "args": {
      "External id": 2487274,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4073
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657551765.598, "dur": 0.824,
    "args": {
      "External id": 2487275,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4074
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657551770.642, "dur": 4.058,
    "args": {
      "External id": 2487276,"Record function id": 0, "Ev Idx": 4075
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657551772.447, "dur": 1.840,
    "args": {
      "External id": 2487277,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4076
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657551772.898, "dur": 0.934,
    "args": {
      "External id": 2487278,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4077
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657551773.169, "dur": 0.584,
    "args": {
      "External id": 2487279,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4078
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657551777.762, "dur": 6.563,
    "args": {
      "External id": 2487280,"Record function id": 0, "Ev Idx": 4079
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657551779.305, "dur": 4.597,
    "args": {
      "External id": 2487281,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4080
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657551780.030, "dur": 3.423,
    "args": {
      "External id": 2487282,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4081
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657551780.730, "dur": 2.626,
    "args": {
      "External id": 2487283,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4082
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657551788.208, "dur": 35390.974,
    "args": {
      "External id": 2487284,"Record function id": 0, "Sequence number": 24740443, "Fwd thread id": 1, "Ev Idx": 4083
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657551789.339, "dur": 35380.464,
    "args": {
      "External id": 2487285,"Sequence number": 24740443, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4084
    }
  },
  {
    "ph": "f", "id": 205, "pid": 1336754, "tid": 1381179, "ts": 1514657551789.339,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.11)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657551821.183, "dur": 35.892,
    "args": {
      "External id": 2487286,"Record function id": 0, "Ev Idx": 4085
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.11)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657551864.736, "dur": 61.505,
    "args": {
      "External id": 2487287,"Record function id": 0, "Ev Idx": 4086
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.11)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657551931.950, "dur": 35230.274,
    "args": {
      "External id": 2487288,"Record function id": 0, "Ev Idx": 4087
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657552062.621, "dur": 7.746,
    "args": {
      "External id": 2487289,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4088
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657552080.751, "dur": 5.121,
    "args": {
      "External id": 2487290,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 4089
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514657552105.245, "dur": 34197.469,
    "args": {
      "External id": 2487291,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4090
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514657552118.601, "dur": 34174.868,
    "args": {
      "External id": 2487292,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4091
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657552188.671, "dur": 14.761,
    "args": {
      "External id": 2487293,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4092
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514657552209.981, "dur": 34043.160,
    "args": {
      "External id": 2487294,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 4093
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657552212.631, "dur": 34039.737,
    "args": {
      "External id": 2487295,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 4094
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657552216.451, "dur": 8.615,
    "args": {
      "External id": 2487296,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4095
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657552248.894, "dur": 34000.079,
    "args": {
      "External id": 2487297,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 4096
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657586395.936, "dur": 9.612,
    "args": {
      "External id": 2487298,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 4097
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657586399.387, "dur": 5.807,
    "args": {
      "External id": 2487299,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4098
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657586435.771, "dur": 384.111,
    "args": {
      "External id": 2487300,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 4099
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657586461.962, "dur": 352.535,
    "args": {
      "External id": 2487301,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4100, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514657586472.916, "dur": 335.506,
    "args": {
      "External id": 2487302,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 4101
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657586840.587, "dur": 2.323,
    "args": {
      "External id": 2487303,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4102, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657586903.037, "dur": 6.595,
    "args": {
      "External id": 2487304,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4103
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657586953.957, "dur": 4.131,
    "args": {
      "External id": 2487305,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4104
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657586975.092, "dur": 1.172,
    "args": {
      "External id": 2487306,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4105
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587024.113, "dur": 1.547,
    "args": {
      "External id": 2487307,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4106
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587039.178, "dur": 1.150,
    "args": {
      "External id": 2487308,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4107
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587052.058, "dur": 2.994,
    "args": {
      "External id": 2487309,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4108
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587072.443, "dur": 1.180,
    "args": {
      "External id": 2487310,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4109
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587086.251, "dur": 1.822,
    "args": {
      "External id": 2487311,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4110
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587098.355, "dur": 0.807,
    "args": {
      "External id": 2487312,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4111
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587195.064, "dur": 2885.551,
    "args": {
      "External id": 2487313,"Record function id": 0, "Ev Idx": 4112
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.10)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587215.749, "dur": 1064.693,
    "args": {
      "External id": 2487314,"Record function id": 0, "Ev Idx": 4113
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.10)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587231.390, "dur": 337.174,
    "args": {
      "External id": 2487315,"Record function id": 0, "Ev Idx": 4114
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587315.603, "dur": 5.868,
    "args": {
      "External id": 2487316,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 4115
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587324.856, "dur": 1.136,
    "args": {
      "External id": 2487317,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 4116
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587327.698, "dur": 1.148,
    "args": {
      "External id": 2487318,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4117
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587330.345, "dur": 1.019,
    "args": {
      "External id": 2487319,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4118
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587332.880, "dur": 0.914,
    "args": {
      "External id": 2487320,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4119
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587335.618, "dur": 0.801,
    "args": {
      "External id": 2487321,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4120
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587337.743, "dur": 1.822,
    "args": {
      "External id": 2487322,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 4121
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587340.837, "dur": 0.999,
    "args": {
      "External id": 2487323,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4122
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587343.253, "dur": 3.308,
    "args": {
      "External id": 2487324,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4123
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587347.907, "dur": 0.691,
    "args": {
      "External id": 2487325,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4124
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587365.971, "dur": 153.752,
    "args": {
      "External id": 2487326,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 4125
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587381.135, "dur": 133.816,
    "args": {
      "External id": 2487327,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 4126
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587398.605, "dur": 12.911,
    "args": {
      "External id": 2487328,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4127
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587414.831, "dur": 69.294,
    "args": {
      "External id": 2487329,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 4128
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587417.640, "dur": 66.110,
    "args": {
      "External id": 2487330,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 4129
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587421.432, "dur": 5.578,
    "args": {
      "External id": 2487331,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4130
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587428.877, "dur": 54.259,
    "args": {
      "External id": 2487332,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 4131
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.9", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587665.162, "dur": 607.583,
    "args": {
      "External id": 2487333,"Record function id": 0, "Ev Idx": 4132
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.9)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587683.013, "dur": 574.096,
    "args": {
      "External id": 2487334,"Record function id": 0, "Ev Idx": 4133
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587749.278, "dur": 5.825,
    "args": {
      "External id": 2487335,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4134
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587771.185, "dur": 33.094,
    "args": {
      "External id": 2487336,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 4135
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587776.101, "dur": 1.701,
    "args": {
      "External id": 2487337,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4136
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587780.162, "dur": 0.422,
    "args": {
      "External id": 2487338,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4137
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587782.098, "dur": 0.646,
    "args": {
      "External id": 2487339,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4138
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587784.211, "dur": 2.183,
    "args": {
      "External id": 2487340,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4139
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587787.860, "dur": 0.365,
    "args": {
      "External id": 2487341,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4140
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587789.655, "dur": 0.426,
    "args": {
      "External id": 2487342,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4141
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587795.758, "dur": 0.853,
    "args": {
      "External id": 2487343,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4142
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587798.508, "dur": 0.449,
    "args": {
      "External id": 2487344,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4143
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587800.279, "dur": 0.604,
    "args": {
      "External id": 2487345,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4144
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587814.532, "dur": 36.560,
    "args": {
      "External id": 2487346,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 4145
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587881.467, "dur": 141.795,
    "args": {
      "External id": 2487347,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "1", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 4146
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587891.355, "dur": 3.201,
    "args": {
      "External id": 2487348,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4147
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587900.086, "dur": 9.653,
    "args": {
      "External id": 2487349,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 4148
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587904.307, "dur": 5.025,
    "args": {
      "External id": 2487350,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "12846080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 4149
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587907.682, "dur": 0.493,
    "args": {
      "External id": 2487351,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 4150
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587918.959, "dur": 26.903,
    "args": {
      "External id": 2487352,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 4151
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587921.006, "dur": 2.709,
    "args": {
      "External id": 2487353,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4152
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587925.213, "dur": 0.560,
    "args": {
      "External id": 2487354,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6423296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4153
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587927.379, "dur": 0.482,
    "args": {
      "External id": 2487355,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6947584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4154
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587929.542, "dur": 0.368,
    "args": {
      "External id": 2487356,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7471872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4155
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587931.902, "dur": 0.555,
    "args": {
      "External id": 2487357,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7996160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4156
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587934.053, "dur": 0.486,
    "args": {
      "External id": 2487358,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8520448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4157
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587935.983, "dur": 0.525,
    "args": {
      "External id": 2487359,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "8520704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4158
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587937.890, "dur": 0.333,
    "args": {
      "External id": 2487360,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "9962496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4159
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587939.869, "dur": 2.350,
    "args": {
      "External id": 2487361,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11404288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4160
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657587955.868, "dur": 21.435,
    "args": {
      "External id": 2487362,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 4161
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657588075.245, "dur": 114.013,
    "args": {
      "External id": 2487363,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 4162
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657588097.363, "dur": 88.288,
    "args": {
      "External id": 2487364,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4163, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514657588107.694, "dur": 74.073,
    "args": {
      "External id": 2487365,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 4164
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657588205.413, "dur": 1.778,
    "args": {
      "External id": 2487366,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4165, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657588287.566, "dur": 1772.276,
    "args": {
      "External id": 2487367,"Sequence number": 24740442, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4166
    }
  },
  {
    "ph": "f", "id": 206, "pid": 1336754, "tid": 1381179, "ts": 1514657588287.566,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657588399.530, "dur": 104.630,
    "args": {
      "External id": 2487368,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 4167
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514657588560.284, "dur": 46.669,
    "args": {
      "External id": 2487369,"kernel_hash": "cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/fu/cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 4168
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657588629.897, "dur": 59.189,
    "args": {
      "External id": 2487370,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 4169
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657588699.838, "dur": 34.464,
    "args": {
      "External id": 2487371,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4170
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657588741.335, "dur": 46.845,
    "args": {
      "External id": 2487372,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4171
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657588796.226, "dur": 29.175,
    "args": {
      "External id": 2487373,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4172
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657588836.240, "dur": 47.052,
    "args": {
      "External id": 2487374,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4173
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657588908.541, "dur": 24.513,
    "args": {
      "External id": 2487375,"kernel_hash": "cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/wm/cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 4174
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514657588968.607, "dur": 72.673,
    "args": {
      "External id": 2487376,"kernel_hash": "cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/br/cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4175
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657589066.592, "dur": 22.186,
    "args": {
      "External id": 2487377,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4176
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514657589102.859, "dur": 17.699,
    "args": {
      "External id": 2487378,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4177
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657589132.124, "dur": 37.183,
    "args": {
      "External id": 2487379,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4178
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657589172.520, "dur": 39.758,
    "args": {
      "External id": 2487380,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4179
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657589243.759, "dur": 172.726,
    "args": {
      "External id": 2487381,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 4180
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657589320.538, "dur": 5.686,
    "args": {
      "External id": 2487382,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4181
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657589328.219, "dur": 2.229,
    "args": {
      "External id": 2487383,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4182
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657589449.767, "dur": 25.278,
    "args": {
      "External id": 2487384,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4183
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657589485.913, "dur": 15.807,
    "args": {
      "External id": 2487385,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4184
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657589509.891, "dur": 54.435,
    "args": {
      "External id": 2487386,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4185
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657589580.029, "dur": 40.216,
    "args": {
      "External id": 2487387,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4186
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657589631.504, "dur": 22.759,
    "args": {
      "External id": 2487388,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4187
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657589675.299, "dur": 43.684,
    "args": {
      "External id": 2487389,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4188
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657589731.130, "dur": 23.205,
    "args": {
      "External id": 2487390,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4189
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657589765.271, "dur": 35.658,
    "args": {
      "External id": 2487391,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4190
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336754, "tid": 1381179,
    "ts": 1514657589819.933, "dur": 25.163,
    "args": {
      "External id": 2487392,"kernel_hash": "c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/6w/c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 4191
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657589865.444, "dur": 24.717,
    "args": {
      "External id": 2487393,"kernel_hash": "c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/4k/c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4192
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657589913.126, "dur": 18.629,
    "args": {
      "External id": 2487394,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4193
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514657589944.889, "dur": 16.497,
    "args": {
      "External id": 2487395,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4194
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336754, "tid": 1381179,
    "ts": 1514657589974.230, "dur": 52.641,
    "args": {
      "External id": 2487396,"kernel_hash": "cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/eb/cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 4195
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657590102.655, "dur": 20.939,
    "args": {
      "External id": 2487397,"Record function id": 0, "Ev Idx": 4196
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657590105.759, "dur": 16.857,
    "args": {
      "External id": 2487398,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4197
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657590110.503, "dur": 11.107,
    "args": {
      "External id": 2487399,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4198
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657590115.534, "dur": 5.962,
    "args": {
      "External id": 2487400,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4199
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657590127.560, "dur": 4.667,
    "args": {
      "External id": 2487401,"Record function id": 0, "Ev Idx": 4200
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657590128.833, "dur": 2.964,
    "args": {
      "External id": 2487402,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4201
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657590129.548, "dur": 1.816,
    "args": {
      "External id": 2487403,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4202
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657590130.276, "dur": 0.996,
    "args": {
      "External id": 2487404,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4203
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657590135.443, "dur": 4.093,
    "args": {
      "External id": 2487405,"Record function id": 0, "Ev Idx": 4204
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657590136.692, "dur": 2.451,
    "args": {
      "External id": 2487406,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4205
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657590137.239, "dur": 1.477,
    "args": {
      "External id": 2487407,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4206
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657590137.729, "dur": 0.889,
    "args": {
      "External id": 2487408,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4207
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657590142.707, "dur": 4.372,
    "args": {
      "External id": 2487409,"Record function id": 0, "Ev Idx": 4208
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657590144.036, "dur": 2.652,
    "args": {
      "External id": 2487410,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4209
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657590145.028, "dur": 1.262,
    "args": {
      "External id": 2487411,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4210
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657590145.524, "dur": 0.702,
    "args": {
      "External id": 2487412,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4211
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657590150.124, "dur": 3.573,
    "args": {
      "External id": 2487413,"Record function id": 0, "Ev Idx": 4212
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657590151.118, "dur": 2.175,
    "args": {
      "External id": 2487414,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4213
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657590151.585, "dur": 1.256,
    "args": {
      "External id": 2487415,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4214
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657590152.060, "dur": 0.715,
    "args": {
      "External id": 2487416,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4215
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657590156.784, "dur": 3.721,
    "args": {
      "External id": 2487417,"Record function id": 0, "Ev Idx": 4216
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657590157.987, "dur": 2.096,
    "args": {
      "External id": 2487418,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4217
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657590158.461, "dur": 1.175,
    "args": {
      "External id": 2487419,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4218
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657590158.916, "dur": 0.612,
    "args": {
      "External id": 2487420,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4219
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657590163.892, "dur": 3.737,
    "args": {
      "External id": 2487421,"Record function id": 0, "Ev Idx": 4220
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657590165.033, "dur": 2.183,
    "args": {
      "External id": 2487422,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4221
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657590165.634, "dur": 1.149,
    "args": {
      "External id": 2487423,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4222
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657590165.914, "dur": 0.786,
    "args": {
      "External id": 2487424,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4223
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657590173.989, "dur": 5.082,
    "args": {
      "External id": 2487425,"Record function id": 0, "Ev Idx": 4224
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657590174.873, "dur": 3.785,
    "args": {
      "External id": 2487426,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4225
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657590175.567, "dur": 2.668,
    "args": {
      "External id": 2487427,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4226
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657590175.833, "dur": 2.340,
    "args": {
      "External id": 2487428,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4227
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657590182.080, "dur": 3.994,
    "args": {
      "External id": 2487429,"Record function id": 0, "Ev Idx": 4228
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657590183.238, "dur": 2.405,
    "args": {
      "External id": 2487430,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4229
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657590183.844, "dur": 1.377,
    "args": {
      "External id": 2487431,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4230
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657590184.286, "dur": 0.852,
    "args": {
      "External id": 2487432,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4231
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657590189.939, "dur": 34919.280,
    "args": {
      "External id": 2487433,"Record function id": 0, "Sequence number": 24740441, "Fwd thread id": 1, "Ev Idx": 4232
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657590191.186, "dur": 34909.127,
    "args": {
      "External id": 2487434,"Sequence number": 24740441, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4233
    }
  },
  {
    "ph": "f", "id": 207, "pid": 1336754, "tid": 1381179, "ts": 1514657590191.186,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.10)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657590220.969, "dur": 37.412,
    "args": {
      "External id": 2487435,"Record function id": 0, "Ev Idx": 4234
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.10)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657590265.615, "dur": 65.045,
    "args": {
      "External id": 2487436,"Record function id": 0, "Ev Idx": 4235
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.10)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657590337.313, "dur": 34754.885,
    "args": {
      "External id": 2487437,"Record function id": 0, "Ev Idx": 4236
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657590424.149, "dur": 6.901,
    "args": {
      "External id": 2487438,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4237
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657590440.484, "dur": 4.603,
    "args": {
      "External id": 2487439,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 4238
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514657590459.881, "dur": 33692.674,
    "args": {
      "External id": 2487440,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4239
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514657590473.813, "dur": 33669.355,
    "args": {
      "External id": 2487441,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4240
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657590547.426, "dur": 15.364,
    "args": {
      "External id": 2487442,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4241
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514657590572.992, "dur": 33527.313,
    "args": {
      "External id": 2487443,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 4242
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657590575.806, "dur": 33523.800,
    "args": {
      "External id": 2487444,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 4243
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657590579.932, "dur": 5.550,
    "args": {
      "External id": 2487445,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4244
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657590587.555, "dur": 33508.531,
    "args": {
      "External id": 2487446,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 4245
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657624245.779, "dur": 9.061,
    "args": {
      "External id": 2487447,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 4246
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657624249.095, "dur": 5.450,
    "args": {
      "External id": 2487448,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4247
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657624283.751, "dur": 460.975,
    "args": {
      "External id": 2487449,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 4248
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657624309.965, "dur": 428.506,
    "args": {
      "External id": 2487450,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4249, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514657624321.319, "dur": 410.394,
    "args": {
      "External id": 2487451,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 4250
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657624766.020, "dur": 2.180,
    "args": {
      "External id": 2487452,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4251, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657624831.787, "dur": 7.122,
    "args": {
      "External id": 2487453,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4252
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657624888.948, "dur": 1.447,
    "args": {
      "External id": 2487454,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4253
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657624906.872, "dur": 1.187,
    "args": {
      "External id": 2487455,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4254
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657624919.756, "dur": 1.047,
    "args": {
      "External id": 2487456,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4255
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657624932.389, "dur": 1.075,
    "args": {
      "External id": 2487457,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4256
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657624946.809, "dur": 0.864,
    "args": {
      "External id": 2487458,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4257
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657624960.423, "dur": 1.246,
    "args": {
      "External id": 2487459,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4258
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657624972.545, "dur": 2.186,
    "args": {
      "External id": 2487460,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4259
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657625021.288, "dur": 1.572,
    "args": {
      "External id": 2487461,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4260
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657625124.536, "dur": 2851.662,
    "args": {
      "External id": 2487462,"Record function id": 0, "Ev Idx": 4261
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.9)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657625144.429, "dur": 1065.974,
    "args": {
      "External id": 2487463,"Record function id": 0, "Ev Idx": 4262
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.9)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657625160.879, "dur": 327.168,
    "args": {
      "External id": 2487464,"Record function id": 0, "Ev Idx": 4263
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657625246.267, "dur": 3.983,
    "args": {
      "External id": 2487465,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 4264
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657625253.263, "dur": 1.504,
    "args": {
      "External id": 2487466,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 4265
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657625256.743, "dur": 1.470,
    "args": {
      "External id": 2487467,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4266
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657625259.634, "dur": 1.296,
    "args": {
      "External id": 2487468,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4267
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657625262.228, "dur": 1.091,
    "args": {
      "External id": 2487469,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4268
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657625264.749, "dur": 1.294,
    "args": {
      "External id": 2487470,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4269
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657625267.642, "dur": 1.558,
    "args": {
      "External id": 2487471,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 4270
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657625270.652, "dur": 3.903,
    "args": {
      "External id": 2487472,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4271
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657625275.736, "dur": 1.627,
    "args": {
      "External id": 2487473,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4272
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657625278.620, "dur": 1.304,
    "args": {
      "External id": 2487474,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4273
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657625297.450, "dur": 159.259,
    "args": {
      "External id": 2487475,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 4274
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657625313.986, "dur": 138.045,
    "args": {
      "External id": 2487476,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 4275
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657625337.262, "dur": 12.384,
    "args": {
      "External id": 2487477,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4276
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514657625353.239, "dur": 70.019,
    "args": {
      "External id": 2487478,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 4277
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657625356.251, "dur": 66.692,
    "args": {
      "External id": 2487479,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 4278
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657625359.959, "dur": 5.616,
    "args": {
      "External id": 2487480,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4279
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657625369.035, "dur": 53.093,
    "args": {
      "External id": 2487481,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 4280
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.8", "pid": 1336754, "tid": 1381179,
    "ts": 1514657625607.715, "dur": 595.275,
    "args": {
      "External id": 2487482,"Record function id": 0, "Ev Idx": 4281
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.8)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657625624.628, "dur": 564.836,
    "args": {
      "External id": 2487483,"Record function id": 0, "Ev Idx": 4282
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657625688.656, "dur": 6.001,
    "args": {
      "External id": 2487484,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4283
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1381179,
    "ts": 1514657625710.425, "dur": 29.614,
    "args": {
      "External id": 2487485,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 4284
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657625715.235, "dur": 1.611,
    "args": {
      "External id": 2487486,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4285
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657625719.051, "dur": 0.478,
    "args": {
      "External id": 2487487,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4286
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657625721.249, "dur": 2.770,
    "args": {
      "External id": 2487488,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4287
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657625725.622, "dur": 0.713,
    "args": {
      "External id": 2487489,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4288
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657625728.212, "dur": 0.433,
    "args": {
      "External id": 2487490,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4289
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657625730.160, "dur": 0.637,
    "args": {
      "External id": 2487491,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4290
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657625732.086, "dur": 0.634,
    "args": {
      "External id": 2487492,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4291
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657625734.091, "dur": 0.515,
    "args": {
      "External id": 2487493,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4292
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657625736.127, "dur": 0.534,
    "args": {
      "External id": 2487494,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4293
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657625750.765, "dur": 34.426,
    "args": {
      "External id": 2487495,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 4294
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1381179,
    "ts": 1514657625815.808, "dur": 106.210,
    "args": {
      "External id": 2487496,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "1", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 4295
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657625826.249, "dur": 2.944,
    "args": {
      "External id": 2487497,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4296
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1381179,
    "ts": 1514657625835.053, "dur": 12.396,
    "args": {
      "External id": 2487498,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 4297
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1381179,
    "ts": 1514657625839.332, "dur": 7.696,
    "args": {
      "External id": 2487499,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "12846080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 4298
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657625842.988, "dur": 2.431,
    "args": {
      "External id": 2487500,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 4299
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1381179,
    "ts": 1514657625854.149, "dur": 28.895,
    "args": {
      "External id": 2487501,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 4300
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657625856.413, "dur": 0.487,
    "args": {
      "External id": 2487502,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4301
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657625858.634, "dur": 0.445,
    "args": {
      "External id": 2487503,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6423296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4302
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657625860.600, "dur": 0.505,
    "args": {
      "External id": 2487504,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6947584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4303
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657625862.670, "dur": 0.422,
    "args": {
      "External id": 2487505,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7471872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4304
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657625864.632, "dur": 0.785,
    "args": {
      "External id": 2487506,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7996160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4305
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657625866.819, "dur": 0.722,
    "args": {
      "External id": 2487507,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8520448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4306
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657625869.142, "dur": 0.550,
    "args": {
      "External id": 2487508,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "8520704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4307
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657625870.996, "dur": 2.302,
    "args": {
      "External id": 2487509,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "9962496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4308
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657625874.819, "dur": 0.428,
    "args": {
      "External id": 2487510,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11404288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4309
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657625892.657, "dur": 21.137,
    "args": {
      "External id": 2487511,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 4310
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657625967.577, "dur": 153.016,
    "args": {
      "External id": 2487512,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 4311
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657626025.149, "dur": 91.585,
    "args": {
      "External id": 2487513,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4312, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514657626037.123, "dur": 75.863,
    "args": {
      "External id": 2487514,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 4313
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657626134.356, "dur": 1.918,
    "args": {
      "External id": 2487515,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4314, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657626218.373, "dur": 1735.851,
    "args": {
      "External id": 2487516,"Sequence number": 24740440, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4315
    }
  },
  {
    "ph": "f", "id": 208, "pid": 1336754, "tid": 1381179, "ts": 1514657626218.373,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657626330.824, "dur": 106.516,
    "args": {
      "External id": 2487517,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 4316
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514657626474.043, "dur": 42.427,
    "args": {
      "External id": 2487518,"kernel_hash": "cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/fu/cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 4317
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657626557.166, "dur": 58.225,
    "args": {
      "External id": 2487519,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 4318
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657626626.495, "dur": 35.372,
    "args": {
      "External id": 2487520,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4319
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657626668.728, "dur": 46.365,
    "args": {
      "External id": 2487521,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4320
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657626723.871, "dur": 29.230,
    "args": {
      "External id": 2487522,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4321
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657626763.343, "dur": 43.217,
    "args": {
      "External id": 2487523,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4322
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657626830.139, "dur": 25.931,
    "args": {
      "External id": 2487524,"kernel_hash": "cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/wm/cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 4323
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514657626874.434, "dur": 31.559,
    "args": {
      "External id": 2487525,"kernel_hash": "cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/br/cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4324
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657626925.847, "dur": 19.510,
    "args": {
      "External id": 2487526,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4325
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514657626958.086, "dur": 16.777,
    "args": {
      "External id": 2487527,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4326
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657627024.244, "dur": 35.984,
    "args": {
      "External id": 2487528,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4327
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657627064.508, "dur": 34.163,
    "args": {
      "External id": 2487529,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4328
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657627131.978, "dur": 189.573,
    "args": {
      "External id": 2487530,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 4329
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657627217.013, "dur": 6.594,
    "args": {
      "External id": 2487531,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4330
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657627225.807, "dur": 2.482,
    "args": {
      "External id": 2487532,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4331
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657627355.294, "dur": 26.232,
    "args": {
      "External id": 2487533,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4332
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657627392.251, "dur": 15.778,
    "args": {
      "External id": 2487534,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4333
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657627417.942, "dur": 36.895,
    "args": {
      "External id": 2487535,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4334
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657627461.968, "dur": 37.942,
    "args": {
      "External id": 2487536,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4335
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657627507.113, "dur": 57.164,
    "args": {
      "External id": 2487537,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4336
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657627602.288, "dur": 47.645,
    "args": {
      "External id": 2487538,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4337
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657627661.012, "dur": 22.810,
    "args": {
      "External id": 2487539,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4338
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657627692.758, "dur": 35.478,
    "args": {
      "External id": 2487540,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4339
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336754, "tid": 1381179,
    "ts": 1514657627747.593, "dur": 26.833,
    "args": {
      "External id": 2487541,"kernel_hash": "c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/6w/c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 4340
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657627792.181, "dur": 27.821,
    "args": {
      "External id": 2487542,"kernel_hash": "c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/4k/c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4341
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657627837.606, "dur": 18.156,
    "args": {
      "External id": 2487543,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4342
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514657627871.102, "dur": 21.650,
    "args": {
      "External id": 2487544,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4343
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336754, "tid": 1381179,
    "ts": 1514657627905.021, "dur": 17.758,
    "args": {
      "External id": 2487545,"kernel_hash": "cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/eb/cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 4344
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657628036.461, "dur": 20.315,
    "args": {
      "External id": 2487546,"Record function id": 0, "Ev Idx": 4345
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657628040.565, "dur": 15.013,
    "args": {
      "External id": 2487547,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4346
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657628045.276, "dur": 8.690,
    "args": {
      "External id": 2487548,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4347
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657628046.900, "dur": 6.795,
    "args": {
      "External id": 2487549,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4348
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657628060.793, "dur": 8.848,
    "args": {
      "External id": 2487550,"Record function id": 0, "Ev Idx": 4349
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657628062.373, "dur": 6.847,
    "args": {
      "External id": 2487551,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4350
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657628063.357, "dur": 5.369,
    "args": {
      "External id": 2487552,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4351
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657628064.532, "dur": 4.068,
    "args": {
      "External id": 2487553,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4352
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657628072.977, "dur": 4.794,
    "args": {
      "External id": 2487554,"Record function id": 0, "Ev Idx": 4353
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657628074.311, "dur": 3.067,
    "args": {
      "External id": 2487555,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4354
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657628075.083, "dur": 1.807,
    "args": {
      "External id": 2487556,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4355
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657628075.591, "dur": 1.224,
    "args": {
      "External id": 2487557,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4356
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657628081.002, "dur": 4.773,
    "args": {
      "External id": 2487558,"Record function id": 0, "Ev Idx": 4357
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657628082.469, "dur": 2.914,
    "args": {
      "External id": 2487559,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4358
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657628083.136, "dur": 1.538,
    "args": {
      "External id": 2487560,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4359
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657628083.648, "dur": 0.948,
    "args": {
      "External id": 2487561,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4360
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657628088.869, "dur": 4.358,
    "args": {
      "External id": 2487562,"Record function id": 0, "Ev Idx": 4361
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657628089.876, "dur": 2.961,
    "args": {
      "External id": 2487563,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4362
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657628090.536, "dur": 1.653,
    "args": {
      "External id": 2487564,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4363
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657628091.235, "dur": 0.889,
    "args": {
      "External id": 2487565,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4364
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657628096.328, "dur": 4.952,
    "args": {
      "External id": 2487566,"Record function id": 0, "Ev Idx": 4365
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657628097.730, "dur": 3.117,
    "args": {
      "External id": 2487567,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4366
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657628098.437, "dur": 1.686,
    "args": {
      "External id": 2487568,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4367
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657628099.154, "dur": 0.856,
    "args": {
      "External id": 2487569,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4368
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657628104.627, "dur": 3.895,
    "args": {
      "External id": 2487570,"Record function id": 0, "Ev Idx": 4369
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657628106.023, "dur": 2.077,
    "args": {
      "External id": 2487571,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4370
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657628106.500, "dur": 0.937,
    "args": {
      "External id": 2487572,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4371
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657628106.784, "dur": 0.553,
    "args": {
      "External id": 2487573,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4372
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657628111.630, "dur": 5.299,
    "args": {
      "External id": 2487574,"Record function id": 0, "Ev Idx": 4373
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657628112.642, "dur": 3.888,
    "args": {
      "External id": 2487575,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4374
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657628113.090, "dur": 2.724,
    "args": {
      "External id": 2487576,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4375
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657628113.364, "dur": 2.374,
    "args": {
      "External id": 2487577,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4376
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657628119.950, "dur": 4.624,
    "args": {
      "External id": 2487578,"Record function id": 0, "Ev Idx": 4377
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657628121.292, "dur": 2.877,
    "args": {
      "External id": 2487579,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4378
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657628121.733, "dur": 1.718,
    "args": {
      "External id": 2487580,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4379
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657628122.457, "dur": 0.893,
    "args": {
      "External id": 2487581,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4380
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657628131.335, "dur": 34742.583,
    "args": {
      "External id": 2487582,"Record function id": 0, "Sequence number": 24740439, "Fwd thread id": 1, "Ev Idx": 4381
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657628132.631, "dur": 34731.992,
    "args": {
      "External id": 2487583,"Sequence number": 24740439, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4382
    }
  },
  {
    "ph": "f", "id": 209, "pid": 1336754, "tid": 1381179, "ts": 1514657628132.631,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.9)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657628162.730, "dur": 38.962,
    "args": {
      "External id": 2487584,"Record function id": 0, "Ev Idx": 4383
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.9)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657628209.952, "dur": 65.088,
    "args": {
      "External id": 2487585,"Record function id": 0, "Ev Idx": 4384
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.9)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657628280.828, "dur": 34576.631,
    "args": {
      "External id": 2487586,"Record function id": 0, "Ev Idx": 4385
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657628372.531, "dur": 6.681,
    "args": {
      "External id": 2487587,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4386
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657628388.823, "dur": 4.780,
    "args": {
      "External id": 2487588,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 4387
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514657628411.093, "dur": 33627.500,
    "args": {
      "External id": 2487589,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4388
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514657628423.882, "dur": 33605.215,
    "args": {
      "External id": 2487590,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4389
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657628470.242, "dur": 14.574,
    "args": {
      "External id": 2487591,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4390
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514657628491.399, "dur": 33462.984,
    "args": {
      "External id": 2487592,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 4391
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657628493.933, "dur": 33459.589,
    "args": {
      "External id": 2487593,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 4392
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657628497.564, "dur": 4.909,
    "args": {
      "External id": 2487594,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4393
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657628504.293, "dur": 33445.528,
    "args": {
      "External id": 2487595,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 4394
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657662128.531, "dur": 9.846,
    "args": {
      "External id": 2487596,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 4395
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657662132.259, "dur": 5.699,
    "args": {
      "External id": 2487597,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4396
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657662168.642, "dur": 386.522,
    "args": {
      "External id": 2487598,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 4397
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657662193.351, "dur": 355.957,
    "args": {
      "External id": 2487599,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4398, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514657662204.904, "dur": 336.989,
    "args": {
      "External id": 2487600,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 4399
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657662575.553, "dur": 2.841,
    "args": {
      "External id": 2487601,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4400, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657662641.319, "dur": 7.174,
    "args": {
      "External id": 2487602,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4401
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657662696.650, "dur": 1.560,
    "args": {
      "External id": 2487603,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4402
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657662713.861, "dur": 1.181,
    "args": {
      "External id": 2487604,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4403
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657662727.054, "dur": 1.081,
    "args": {
      "External id": 2487605,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4404
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657662739.489, "dur": 1.073,
    "args": {
      "External id": 2487606,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4405
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657662753.809, "dur": 0.947,
    "args": {
      "External id": 2487607,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4406
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657662766.549, "dur": 1.122,
    "args": {
      "External id": 2487608,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4407
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657662778.961, "dur": 1.277,
    "args": {
      "External id": 2487609,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4408
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657662790.835, "dur": 1.489,
    "args": {
      "External id": 2487610,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4409
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657662888.694, "dur": 2856.478,
    "args": {
      "External id": 2487611,"Record function id": 0, "Ev Idx": 4410
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.8)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657662908.468, "dur": 1050.392,
    "args": {
      "External id": 2487612,"Record function id": 0, "Ev Idx": 4411
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.8)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657662923.668, "dur": 364.593,
    "args": {
      "External id": 2487613,"Record function id": 0, "Ev Idx": 4412
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657663045.859, "dur": 5.376,
    "args": {
      "External id": 2487614,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 4413
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657663054.780, "dur": 1.475,
    "args": {
      "External id": 2487615,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 4414
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657663057.982, "dur": 1.400,
    "args": {
      "External id": 2487616,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4415
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657663061.553, "dur": 1.383,
    "args": {
      "External id": 2487617,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4416
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657663064.420, "dur": 1.371,
    "args": {
      "External id": 2487618,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4417
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657663067.483, "dur": 1.374,
    "args": {
      "External id": 2487619,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4418
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657663070.656, "dur": 1.665,
    "args": {
      "External id": 2487620,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 4419
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657663074.398, "dur": 3.238,
    "args": {
      "External id": 2487621,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4420
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657663079.689, "dur": 1.251,
    "args": {
      "External id": 2487622,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4421
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657663082.623, "dur": 1.790,
    "args": {
      "External id": 2487623,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4422
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657663103.253, "dur": 154.867,
    "args": {
      "External id": 2487624,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 4423
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657663120.638, "dur": 132.735,
    "args": {
      "External id": 2487625,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 4424
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657663139.226, "dur": 12.530,
    "args": {
      "External id": 2487626,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4425
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514657663157.192, "dur": 67.591,
    "args": {
      "External id": 2487627,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 4426
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657663159.934, "dur": 64.509,
    "args": {
      "External id": 2487628,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 4427
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657663163.583, "dur": 5.391,
    "args": {
      "External id": 2487629,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4428
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657663170.562, "dur": 53.344,
    "args": {
      "External id": 2487630,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 4429
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.7", "pid": 1336754, "tid": 1381179,
    "ts": 1514657663379.570, "dur": 571.017,
    "args": {
      "External id": 2487631,"Record function id": 0, "Ev Idx": 4430
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.7)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657663396.165, "dur": 542.931,
    "args": {
      "External id": 2487632,"Record function id": 0, "Ev Idx": 4431
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657663459.249, "dur": 4.672,
    "args": {
      "External id": 2487633,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4432
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1381179,
    "ts": 1514657663479.611, "dur": 36.909,
    "args": {
      "External id": 2487634,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 4433
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657663488.337, "dur": 1.928,
    "args": {
      "External id": 2487635,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4434
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657663492.599, "dur": 0.910,
    "args": {
      "External id": 2487636,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4435
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657663494.936, "dur": 2.495,
    "args": {
      "External id": 2487637,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4436
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657663499.160, "dur": 0.387,
    "args": {
      "External id": 2487638,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4437
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657663501.214, "dur": 0.437,
    "args": {
      "External id": 2487639,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4438
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657663503.369, "dur": 0.648,
    "args": {
      "External id": 2487640,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4439
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657663505.576, "dur": 0.335,
    "args": {
      "External id": 2487641,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4440
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657663507.602, "dur": 0.411,
    "args": {
      "External id": 2487642,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4441
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657663509.579, "dur": 0.587,
    "args": {
      "External id": 2487643,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4442
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657663527.365, "dur": 52.152,
    "args": {
      "External id": 2487644,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 4443
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1381179,
    "ts": 1514657663614.761, "dur": 105.768,
    "args": {
      "External id": 2487645,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "1", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 4444
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657663625.754, "dur": 3.821,
    "args": {
      "External id": 2487646,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4445
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1381179,
    "ts": 1514657663634.549, "dur": 12.737,
    "args": {
      "External id": 2487647,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 4446
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1381179,
    "ts": 1514657663638.973, "dur": 7.883,
    "args": {
      "External id": 2487648,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "12846080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 4447
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657663642.647, "dur": 2.753,
    "args": {
      "External id": 2487649,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 4448
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1381179,
    "ts": 1514657663654.864, "dur": 23.130,
    "args": {
      "External id": 2487650,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 4449
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657663657.369, "dur": 0.762,
    "args": {
      "External id": 2487651,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4450
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657663659.859, "dur": 0.290,
    "args": {
      "External id": 2487652,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6423296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4451
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657663661.699, "dur": 0.483,
    "args": {
      "External id": 2487653,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6947584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4452
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657663663.422, "dur": 0.633,
    "args": {
      "External id": 2487654,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7471872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4453
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657663665.399, "dur": 0.285,
    "args": {
      "External id": 2487655,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7996160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4454
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657663667.052, "dur": 0.584,
    "args": {
      "External id": 2487656,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8520448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4455
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657663668.865, "dur": 0.482,
    "args": {
      "External id": 2487657,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "8520704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4456
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657663670.771, "dur": 2.366,
    "args": {
      "External id": 2487658,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "9962496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4457
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657663674.546, "dur": 0.412,
    "args": {
      "External id": 2487659,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11404288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4458
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657663693.582, "dur": 19.547,
    "args": {
      "External id": 2487660,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 4459
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657663764.647, "dur": 110.058,
    "args": {
      "External id": 2487661,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 4460
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657663791.405, "dur": 79.737,
    "args": {
      "External id": 2487662,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4461, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514657663801.012, "dur": 66.160,
    "args": {
      "External id": 2487663,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 4462
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657663890.231, "dur": 1.970,
    "args": {
      "External id": 2487664,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4463, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657663965.504, "dur": 1754.240,
    "args": {
      "External id": 2487665,"Sequence number": 24740438, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4464
    }
  },
  {
    "ph": "f", "id": 210, "pid": 1336754, "tid": 1381179, "ts": 1514657663965.504,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657664120.230, "dur": 110.892,
    "args": {
      "External id": 2487666,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 4465
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514657664276.062, "dur": 41.477,
    "args": {
      "External id": 2487667,"kernel_hash": "cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/fu/cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 4466
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657664337.199, "dur": 51.723,
    "args": {
      "External id": 2487668,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 4467
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657664399.072, "dur": 33.408,
    "args": {
      "External id": 2487669,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4468
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657664439.225, "dur": 44.957,
    "args": {
      "External id": 2487670,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4469
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657664491.143, "dur": 27.985,
    "args": {
      "External id": 2487671,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4470
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657664546.568, "dur": 48.009,
    "args": {
      "External id": 2487672,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4471
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657664621.804, "dur": 26.108,
    "args": {
      "External id": 2487673,"kernel_hash": "cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/wm/cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 4472
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514657664667.591, "dur": 28.257,
    "args": {
      "External id": 2487674,"kernel_hash": "cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/br/cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4473
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657664716.047, "dur": 21.412,
    "args": {
      "External id": 2487675,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4474
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514657664752.618, "dur": 16.587,
    "args": {
      "External id": 2487676,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4475
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657664780.503, "dur": 29.661,
    "args": {
      "External id": 2487677,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4476
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657664813.198, "dur": 32.923,
    "args": {
      "External id": 2487678,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4477
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657664874.530, "dur": 218.269,
    "args": {
      "External id": 2487679,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 4478
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657664954.531, "dur": 6.657,
    "args": {
      "External id": 2487680,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4479
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657664963.164, "dur": 2.186,
    "args": {
      "External id": 2487681,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4480
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657665130.169, "dur": 27.355,
    "args": {
      "External id": 2487682,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4481
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657665168.234, "dur": 17.263,
    "args": {
      "External id": 2487683,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4482
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657665195.168, "dur": 55.892,
    "args": {
      "External id": 2487684,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4483
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657665261.227, "dur": 44.044,
    "args": {
      "External id": 2487685,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4484
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657665312.430, "dur": 31.945,
    "args": {
      "External id": 2487686,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4485
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657665355.170, "dur": 42.633,
    "args": {
      "External id": 2487687,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4486
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657665406.156, "dur": 23.101,
    "args": {
      "External id": 2487688,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4487
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657665443.033, "dur": 31.987,
    "args": {
      "External id": 2487689,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4488
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336754, "tid": 1381179,
    "ts": 1514657665496.656, "dur": 24.625,
    "args": {
      "External id": 2487690,"kernel_hash": "c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/6w/c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 4489
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657665560.242, "dur": 28.328,
    "args": {
      "External id": 2487691,"kernel_hash": "c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/4k/c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4490
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657665604.466, "dur": 21.141,
    "args": {
      "External id": 2487692,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4491
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514657665640.571, "dur": 16.570,
    "args": {
      "External id": 2487693,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4492
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336754, "tid": 1381179,
    "ts": 1514657665670.610, "dur": 21.076,
    "args": {
      "External id": 2487694,"kernel_hash": "cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/eb/cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 4493
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657665767.746, "dur": 15.725,
    "args": {
      "External id": 2487695,"Record function id": 0, "Ev Idx": 4494
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657665771.245, "dur": 11.234,
    "args": {
      "External id": 2487696,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4495
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657665775.447, "dur": 6.145,
    "args": {
      "External id": 2487697,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4496
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657665777.328, "dur": 4.124,
    "args": {
      "External id": 2487698,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4497
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657665787.545, "dur": 7.812,
    "args": {
      "External id": 2487699,"Record function id": 0, "Ev Idx": 4498
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657665789.003, "dur": 5.929,
    "args": {
      "External id": 2487700,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4499
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657665790.369, "dur": 4.063,
    "args": {
      "External id": 2487701,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4500
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657665791.242, "dur": 3.119,
    "args": {
      "External id": 2487702,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4501
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657665801.939, "dur": 4.402,
    "args": {
      "External id": 2487703,"Record function id": 0, "Ev Idx": 4502
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657665803.298, "dur": 2.630,
    "args": {
      "External id": 2487704,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4503
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657665804.107, "dur": 1.375,
    "args": {
      "External id": 2487705,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4504
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657665804.499, "dur": 0.914,
    "args": {
      "External id": 2487706,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4505
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657665809.632, "dur": 4.806,
    "args": {
      "External id": 2487707,"Record function id": 0, "Ev Idx": 4506
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657665810.887, "dur": 3.152,
    "args": {
      "External id": 2487708,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4507
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657665811.995, "dur": 1.440,
    "args": {
      "External id": 2487709,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4508
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657665812.615, "dur": 0.748,
    "args": {
      "External id": 2487710,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4509
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657665817.506, "dur": 5.036,
    "args": {
      "External id": 2487711,"Record function id": 0, "Ev Idx": 4510
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657665818.853, "dur": 3.269,
    "args": {
      "External id": 2487712,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4511
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657665819.854, "dur": 1.874,
    "args": {
      "External id": 2487713,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4512
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657665820.559, "dur": 1.103,
    "args": {
      "External id": 2487714,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4513
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657665825.754, "dur": 4.628,
    "args": {
      "External id": 2487715,"Record function id": 0, "Ev Idx": 4514
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657665827.234, "dur": 2.727,
    "args": {
      "External id": 2487716,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4515
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657665828.103, "dur": 1.405,
    "args": {
      "External id": 2487717,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4516
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657665828.779, "dur": 0.628,
    "args": {
      "External id": 2487718,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4517
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657665833.614, "dur": 4.102,
    "args": {
      "External id": 2487719,"Record function id": 0, "Ev Idx": 4518
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657665834.647, "dur": 2.662,
    "args": {
      "External id": 2487720,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4519
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657665835.475, "dur": 1.384,
    "args": {
      "External id": 2487721,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4520
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657665835.972, "dur": 0.823,
    "args": {
      "External id": 2487722,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4521
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657665840.765, "dur": 3.689,
    "args": {
      "External id": 2487723,"Record function id": 0, "Ev Idx": 4522
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657665841.704, "dur": 2.320,
    "args": {
      "External id": 2487724,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4523
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657665842.178, "dur": 1.450,
    "args": {
      "External id": 2487725,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4524
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657665842.801, "dur": 0.728,
    "args": {
      "External id": 2487726,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4525
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657665847.439, "dur": 6.950,
    "args": {
      "External id": 2487727,"Record function id": 0, "Ev Idx": 4526
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657665848.626, "dur": 5.333,
    "args": {
      "External id": 2487728,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4527
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657665849.699, "dur": 3.832,
    "args": {
      "External id": 2487729,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4528
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657665850.205, "dur": 3.214,
    "args": {
      "External id": 2487730,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4529
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657665861.731, "dur": 36208.971,
    "args": {
      "External id": 2487731,"Record function id": 0, "Sequence number": 24740437, "Fwd thread id": 1, "Ev Idx": 4530
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657665862.954, "dur": 36198.445,
    "args": {
      "External id": 2487732,"Sequence number": 24740437, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4531
    }
  },
  {
    "ph": "f", "id": 211, "pid": 1336754, "tid": 1381179, "ts": 1514657665862.954,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.8)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657665892.803, "dur": 42.893,
    "args": {
      "External id": 2487733,"Record function id": 0, "Ev Idx": 4532
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.8)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657665944.611, "dur": 109.792,
    "args": {
      "External id": 2487734,"Record function id": 0, "Ev Idx": 4533
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.8)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657666062.602, "dur": 35989.451,
    "args": {
      "External id": 2487735,"Record function id": 0, "Ev Idx": 4534
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657666157.967, "dur": 7.134,
    "args": {
      "External id": 2487736,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4535
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657666174.855, "dur": 5.394,
    "args": {
      "External id": 2487737,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 4536
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514657666194.872, "dur": 34965.918,
    "args": {
      "External id": 2487738,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4537
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514657666209.161, "dur": 34942.213,
    "args": {
      "External id": 2487739,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4538
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657666258.478, "dur": 15.073,
    "args": {
      "External id": 2487740,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4539
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514657666279.642, "dur": 34832.640,
    "args": {
      "External id": 2487741,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 4540
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657666282.263, "dur": 34829.245,
    "args": {
      "External id": 2487742,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 4541
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657666286.182, "dur": 4.467,
    "args": {
      "External id": 2487743,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4542
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657666292.463, "dur": 34815.298,
    "args": {
      "External id": 2487744,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 4543
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657701251.010, "dur": 11.164,
    "args": {
      "External id": 2487745,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 4544
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657701255.904, "dur": 5.850,
    "args": {
      "External id": 2487746,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4545
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657701291.908, "dur": 426.083,
    "args": {
      "External id": 2487747,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 4546
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657701320.873, "dur": 391.255,
    "args": {
      "External id": 2487748,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4547, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514657701331.944, "dur": 373.965,
    "args": {
      "External id": 2487749,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 4548
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657701741.310, "dur": 2.240,
    "args": {
      "External id": 2487750,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4549, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657701803.441, "dur": 7.135,
    "args": {
      "External id": 2487751,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4550
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657701855.478, "dur": 1.556,
    "args": {
      "External id": 2487752,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4551
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657701874.382, "dur": 1.576,
    "args": {
      "External id": 2487753,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4552
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657701889.049, "dur": 1.169,
    "args": {
      "External id": 2487754,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4553
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657701907.148, "dur": 1.164,
    "args": {
      "External id": 2487755,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4554
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657701918.842, "dur": 1.017,
    "args": {
      "External id": 2487756,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4555
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657701934.154, "dur": 0.990,
    "args": {
      "External id": 2487757,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4556
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657701946.962, "dur": 0.865,
    "args": {
      "External id": 2487758,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4557
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657701957.725, "dur": 0.877,
    "args": {
      "External id": 2487759,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4558
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657702087.229, "dur": 2840.208,
    "args": {
      "External id": 2487760,"Record function id": 0, "Ev Idx": 4559
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.7)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657702107.486, "dur": 1083.765,
    "args": {
      "External id": 2487761,"Record function id": 0, "Ev Idx": 4560
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.7)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657702123.442, "dur": 323.946,
    "args": {
      "External id": 2487762,"Record function id": 0, "Ev Idx": 4561
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657702210.546, "dur": 4.816,
    "args": {
      "External id": 2487763,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 4562
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657702219.893, "dur": 1.439,
    "args": {
      "External id": 2487764,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 4563
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657702222.968, "dur": 3.025,
    "args": {
      "External id": 2487765,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4564
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657702227.843, "dur": 1.180,
    "args": {
      "External id": 2487766,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4565
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657702230.803, "dur": 0.990,
    "args": {
      "External id": 2487767,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4566
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657702234.268, "dur": 0.931,
    "args": {
      "External id": 2487768,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4567
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657702236.587, "dur": 1.459,
    "args": {
      "External id": 2487769,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 4568
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657702239.636, "dur": 1.178,
    "args": {
      "External id": 2487770,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4569
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657702242.285, "dur": 0.944,
    "args": {
      "External id": 2487771,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4570
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657702246.106, "dur": 1.025,
    "args": {
      "External id": 2487772,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4571
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657702265.334, "dur": 154.082,
    "args": {
      "External id": 2487773,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 4572
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657702281.963, "dur": 132.762,
    "args": {
      "External id": 2487774,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 4573
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657702298.243, "dur": 15.319,
    "args": {
      "External id": 2487775,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4574
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514657702316.953, "dur": 68.043,
    "args": {
      "External id": 2487776,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 4575
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657702319.558, "dur": 65.093,
    "args": {
      "External id": 2487777,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 4576
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657702323.875, "dur": 6.300,
    "args": {
      "External id": 2487778,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4577
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657702331.781, "dur": 52.309,
    "args": {
      "External id": 2487779,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 4578
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.6", "pid": 1336754, "tid": 1381179,
    "ts": 1514657702555.980, "dur": 627.793,
    "args": {
      "External id": 2487780,"Record function id": 0, "Ev Idx": 4579
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.6)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657702571.840, "dur": 598.774,
    "args": {
      "External id": 2487781,"Record function id": 0, "Ev Idx": 4580
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657702639.009, "dur": 6.404,
    "args": {
      "External id": 2487782,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4581
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1381179,
    "ts": 1514657702665.953, "dur": 32.817,
    "args": {
      "External id": 2487783,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 4582
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657702670.932, "dur": 3.033,
    "args": {
      "External id": 2487784,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4583
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657702676.029, "dur": 0.895,
    "args": {
      "External id": 2487785,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4584
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657702678.513, "dur": 0.817,
    "args": {
      "External id": 2487786,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4585
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657702681.690, "dur": 0.434,
    "args": {
      "External id": 2487787,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4586
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657702683.488, "dur": 0.818,
    "args": {
      "External id": 2487788,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4587
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657702685.709, "dur": 2.754,
    "args": {
      "External id": 2487789,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4588
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657702690.074, "dur": 0.559,
    "args": {
      "External id": 2487790,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4589
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657702692.069, "dur": 0.668,
    "args": {
      "External id": 2487791,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4590
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657702694.255, "dur": 0.662,
    "args": {
      "External id": 2487792,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4591
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657702709.580, "dur": 34.186,
    "args": {
      "External id": 2487793,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 4592
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1381179,
    "ts": 1514657702774.464, "dur": 122.638,
    "args": {
      "External id": 2487794,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "1", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 4593
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657702784.501, "dur": 3.711,
    "args": {
      "External id": 2487795,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4594
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1381179,
    "ts": 1514657702793.177, "dur": 12.109,
    "args": {
      "External id": 2487796,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 4595
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1381179,
    "ts": 1514657702799.069, "dur": 5.806,
    "args": {
      "External id": 2487797,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "12846080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 4596
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657702802.530, "dur": 0.974,
    "args": {
      "External id": 2487798,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 4597
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1381179,
    "ts": 1514657702812.072, "dur": 26.286,
    "args": {
      "External id": 2487799,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 4598
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657702814.145, "dur": 0.628,
    "args": {
      "External id": 2487800,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4599
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657702816.777, "dur": 0.619,
    "args": {
      "External id": 2487801,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6423296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4600
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657702818.712, "dur": 2.677,
    "args": {
      "External id": 2487802,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6947584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4601
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657702823.291, "dur": 0.628,
    "args": {
      "External id": 2487803,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7471872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4602
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657702825.297, "dur": 0.609,
    "args": {
      "External id": 2487804,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7996160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4603
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657702827.764, "dur": 0.371,
    "args": {
      "External id": 2487805,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8520448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4604
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657702829.804, "dur": 0.341,
    "args": {
      "External id": 2487806,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "8520704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4605
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657702831.563, "dur": 0.357,
    "args": {
      "External id": 2487807,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "9962496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4606
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657702833.108, "dur": 0.562,
    "args": {
      "External id": 2487808,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11404288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4607
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657702868.820, "dur": 20.409,
    "args": {
      "External id": 2487809,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 4608
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657702941.679, "dur": 156.358,
    "args": {
      "External id": 2487810,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 4609
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657702967.517, "dur": 126.791,
    "args": {
      "External id": 2487811,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4610, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514657702976.956, "dur": 112.844,
    "args": {
      "External id": 2487812,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 4611
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657703114.842, "dur": 2.005,
    "args": {
      "External id": 2487813,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4612, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657703198.574, "dur": 1707.200,
    "args": {
      "External id": 2487814,"Sequence number": 24740436, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4613
    }
  },
  {
    "ph": "f", "id": 212, "pid": 1336754, "tid": 1381179, "ts": 1514657703198.574,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657703312.202, "dur": 105.766,
    "args": {
      "External id": 2487815,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 4614
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514657703457.762, "dur": 42.119,
    "args": {
      "External id": 2487816,"kernel_hash": "cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/fu/cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 4615
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657703516.445, "dur": 73.617,
    "args": {
      "External id": 2487817,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 4616
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657703603.917, "dur": 35.200,
    "args": {
      "External id": 2487818,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4617
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657703651.913, "dur": 45.557,
    "args": {
      "External id": 2487819,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4618
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657703704.671, "dur": 28.036,
    "args": {
      "External id": 2487820,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4619
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657703741.439, "dur": 42.036,
    "args": {
      "External id": 2487821,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4620
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657703809.944, "dur": 24.768,
    "args": {
      "External id": 2487822,"kernel_hash": "cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/wm/cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 4621
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514657703857.393, "dur": 30.549,
    "args": {
      "External id": 2487823,"kernel_hash": "cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/br/cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4622
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657703907.946, "dur": 22.813,
    "args": {
      "External id": 2487824,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4623
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514657703944.317, "dur": 16.468,
    "args": {
      "External id": 2487825,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4624
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657703969.981, "dur": 69.319,
    "args": {
      "External id": 2487826,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4625
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657704045.502, "dur": 39.502,
    "args": {
      "External id": 2487827,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4626
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657704118.551, "dur": 174.875,
    "args": {
      "External id": 2487828,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 4627
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657704201.207, "dur": 6.549,
    "args": {
      "External id": 2487829,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4628
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657704209.424, "dur": 2.450,
    "args": {
      "External id": 2487830,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4629
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657704332.986, "dur": 38.062,
    "args": {
      "External id": 2487831,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4630
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657704384.507, "dur": 18.458,
    "args": {
      "External id": 2487832,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4631
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657704410.578, "dur": 34.954,
    "args": {
      "External id": 2487833,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4632
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657704451.209, "dur": 34.341,
    "args": {
      "External id": 2487834,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4633
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657704494.640, "dur": 55.176,
    "args": {
      "External id": 2487835,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4634
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657704566.127, "dur": 46.383,
    "args": {
      "External id": 2487836,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4635
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657704620.614, "dur": 21.101,
    "args": {
      "External id": 2487837,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4636
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657704649.557, "dur": 33.071,
    "args": {
      "External id": 2487838,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4637
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336754, "tid": 1381179,
    "ts": 1514657704704.405, "dur": 27.373,
    "args": {
      "External id": 2487839,"kernel_hash": "c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/6w/c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 4638
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657704749.134, "dur": 26.043,
    "args": {
      "External id": 2487840,"kernel_hash": "c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/4k/c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4639
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657704794.547, "dur": 18.093,
    "args": {
      "External id": 2487841,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4640
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514657704829.846, "dur": 18.067,
    "args": {
      "External id": 2487842,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4641
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336754, "tid": 1381179,
    "ts": 1514657704859.369, "dur": 17.159,
    "args": {
      "External id": 2487843,"kernel_hash": "cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/eb/cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 4642
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657704949.311, "dur": 17.883,
    "args": {
      "External id": 2487844,"Record function id": 0, "Ev Idx": 4643
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657704952.423, "dur": 13.868,
    "args": {
      "External id": 2487845,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4644
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657704959.709, "dur": 5.670,
    "args": {
      "External id": 2487846,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4645
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657704961.303, "dur": 3.978,
    "args": {
      "External id": 2487847,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4646
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657704971.206, "dur": 5.149,
    "args": {
      "External id": 2487848,"Record function id": 0, "Ev Idx": 4647
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657704972.852, "dur": 3.084,
    "args": {
      "External id": 2487849,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4648
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657704973.804, "dur": 1.333,
    "args": {
      "External id": 2487850,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4649
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657704974.144, "dur": 0.924,
    "args": {
      "External id": 2487851,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4650
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657704979.541, "dur": 43.453,
    "args": {
      "External id": 2487852,"Record function id": 0, "Ev Idx": 4651
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657705016.736, "dur": 5.233,
    "args": {
      "External id": 2487853,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4652
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657705018.276, "dur": 2.587,
    "args": {
      "External id": 2487854,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4653
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657705018.958, "dur": 1.667,
    "args": {
      "External id": 2487855,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4654
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657705028.025, "dur": 5.568,
    "args": {
      "External id": 2487856,"Record function id": 0, "Ev Idx": 4655
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657705029.457, "dur": 3.658,
    "args": {
      "External id": 2487857,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4656
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657705030.685, "dur": 1.832,
    "args": {
      "External id": 2487858,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4657
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657705031.585, "dur": 0.824,
    "args": {
      "External id": 2487859,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4658
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657705036.861, "dur": 5.544,
    "args": {
      "External id": 2487860,"Record function id": 0, "Ev Idx": 4659
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657705037.995, "dur": 3.990,
    "args": {
      "External id": 2487861,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4660
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657705038.537, "dur": 2.800,
    "args": {
      "External id": 2487862,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4661
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657705038.842, "dur": 2.419,
    "args": {
      "External id": 2487863,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4662
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657705045.595, "dur": 4.797,
    "args": {
      "External id": 2487864,"Record function id": 0, "Ev Idx": 4663
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657705047.012, "dur": 2.947,
    "args": {
      "External id": 2487865,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4664
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657705048.127, "dur": 1.385,
    "args": {
      "External id": 2487866,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4665
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657705048.418, "dur": 0.976,
    "args": {
      "External id": 2487867,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4666
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657705053.752, "dur": 4.050,
    "args": {
      "External id": 2487868,"Record function id": 0, "Ev Idx": 4667
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657705054.844, "dur": 2.545,
    "args": {
      "External id": 2487869,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4668
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657705055.370, "dur": 1.598,
    "args": {
      "External id": 2487870,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4669
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657705056.154, "dur": 0.705,
    "args": {
      "External id": 2487871,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4670
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657705060.856, "dur": 3.965,
    "args": {
      "External id": 2487872,"Record function id": 0, "Ev Idx": 4671
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657705062.221, "dur": 2.181,
    "args": {
      "External id": 2487873,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4672
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657705062.845, "dur": 1.019,
    "args": {
      "External id": 2487874,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4673
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657705063.109, "dur": 0.685,
    "args": {
      "External id": 2487875,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4674
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657705067.906, "dur": 5.000,
    "args": {
      "External id": 2487876,"Record function id": 0, "Ev Idx": 4675
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657705069.079, "dur": 3.401,
    "args": {
      "External id": 2487877,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4676
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657705069.543, "dur": 2.397,
    "args": {
      "External id": 2487878,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4677
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657705071.163, "dur": 0.707,
    "args": {
      "External id": 2487879,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4678
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657705076.606, "dur": 34787.446,
    "args": {
      "External id": 2487880,"Record function id": 0, "Sequence number": 24740435, "Fwd thread id": 1, "Ev Idx": 4679
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657705077.774, "dur": 34777.686,
    "args": {
      "External id": 2487881,"Sequence number": 24740435, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4680
    }
  },
  {
    "ph": "f", "id": 213, "pid": 1336754, "tid": 1381179, "ts": 1514657705077.774,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.7)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657705106.876, "dur": 35.690,
    "args": {
      "External id": 2487882,"Record function id": 0, "Ev Idx": 4681
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.7)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657705151.165, "dur": 72.801,
    "args": {
      "External id": 2487883,"Record function id": 0, "Ev Idx": 4682
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.7)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657705229.116, "dur": 34619.133,
    "args": {
      "External id": 2487884,"Record function id": 0, "Ev Idx": 4683
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657705319.285, "dur": 6.792,
    "args": {
      "External id": 2487885,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4684
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657705335.129, "dur": 8.293,
    "args": {
      "External id": 2487886,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 4685
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514657705357.430, "dur": 33788.772,
    "args": {
      "External id": 2487887,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4686
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514657705370.411, "dur": 33766.474,
    "args": {
      "External id": 2487888,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4687
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657705410.880, "dur": 14.726,
    "args": {
      "External id": 2487889,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4688
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514657705432.058, "dur": 33664.171,
    "args": {
      "External id": 2487890,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 4689
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657705434.409, "dur": 33661.136,
    "args": {
      "External id": 2487891,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 4690
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657705438.393, "dur": 7.321,
    "args": {
      "External id": 2487892,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4691
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657705447.379, "dur": 33644.289,
    "args": {
      "External id": 2487893,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 4692
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657739232.906, "dur": 10.284,
    "args": {
      "External id": 2487894,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 4693
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657739235.520, "dur": 7.208,
    "args": {
      "External id": 2487895,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4694
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657739272.432, "dur": 293.423,
    "args": {
      "External id": 2487896,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 4695
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657739301.658, "dur": 259.204,
    "args": {
      "External id": 2487897,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4696, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514657739312.860, "dur": 242.173,
    "args": {
      "External id": 2487898,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 4697
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657739586.976, "dur": 2.223,
    "args": {
      "External id": 2487899,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4698, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657739644.839, "dur": 6.505,
    "args": {
      "External id": 2487900,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4699
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657739694.156, "dur": 1.772,
    "args": {
      "External id": 2487901,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4700
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657739711.815, "dur": 1.368,
    "args": {
      "External id": 2487902,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4701
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657739724.194, "dur": 3.429,
    "args": {
      "External id": 2487903,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4702
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657739740.725, "dur": 1.474,
    "args": {
      "External id": 2487904,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4703
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657739753.467, "dur": 1.054,
    "args": {
      "External id": 2487905,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4704
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657739765.030, "dur": 0.956,
    "args": {
      "External id": 2487906,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4705
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657739776.298, "dur": 3.456,
    "args": {
      "External id": 2487907,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4706
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657739789.050, "dur": 1.091,
    "args": {
      "External id": 2487908,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4707
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657739877.257, "dur": 2808.958,
    "args": {
      "External id": 2487909,"Record function id": 0, "Ev Idx": 4708
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.6)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657739896.040, "dur": 1044.854,
    "args": {
      "External id": 2487910,"Record function id": 0, "Ev Idx": 4709
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.6)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657739910.181, "dur": 354.746,
    "args": {
      "External id": 2487911,"Record function id": 0, "Ev Idx": 4710
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657740028.734, "dur": 4.910,
    "args": {
      "External id": 2487912,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 4711
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657740037.159, "dur": 1.400,
    "args": {
      "External id": 2487913,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 4712
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657740042.073, "dur": 1.121,
    "args": {
      "External id": 2487914,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4713
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657740045.016, "dur": 0.989,
    "args": {
      "External id": 2487915,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4714
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657740047.528, "dur": 1.022,
    "args": {
      "External id": 2487916,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4715
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657740050.173, "dur": 3.177,
    "args": {
      "External id": 2487917,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4716
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657740055.588, "dur": 1.215,
    "args": {
      "External id": 2487918,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 4717
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657740058.416, "dur": 1.033,
    "args": {
      "External id": 2487919,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4718
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657740061.162, "dur": 0.952,
    "args": {
      "External id": 2487920,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4719
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657740063.617, "dur": 1.182,
    "args": {
      "External id": 2487921,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4720
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657740084.304, "dur": 151.329,
    "args": {
      "External id": 2487922,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 4721
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657740100.263, "dur": 130.339,
    "args": {
      "External id": 2487923,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 4722
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657740118.090, "dur": 12.498,
    "args": {
      "External id": 2487924,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4723
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514657740133.559, "dur": 68.872,
    "args": {
      "External id": 2487925,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 4724
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657740135.820, "dur": 66.250,
    "args": {
      "External id": 2487926,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 4725
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657740142.448, "dur": 5.693,
    "args": {
      "External id": 2487927,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4726
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657740149.813, "dur": 51.689,
    "args": {
      "External id": 2487928,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 4727
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.5", "pid": 1336754, "tid": 1381179,
    "ts": 1514657740350.670, "dur": 583.114,
    "args": {
      "External id": 2487929,"Record function id": 0, "Ev Idx": 4728
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.5)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657740367.634, "dur": 553.926,
    "args": {
      "External id": 2487930,"Record function id": 0, "Ev Idx": 4729
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657740429.934, "dur": 4.791,
    "args": {
      "External id": 2487931,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4730
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1381179,
    "ts": 1514657740453.694, "dur": 30.617,
    "args": {
      "External id": 2487932,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 4731
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657740457.857, "dur": 3.510,
    "args": {
      "External id": 2487933,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4732
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657740463.812, "dur": 0.692,
    "args": {
      "External id": 2487934,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4733
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657740466.894, "dur": 0.567,
    "args": {
      "External id": 2487935,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4734
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657740468.750, "dur": 0.787,
    "args": {
      "External id": 2487936,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4735
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657740471.184, "dur": 1.039,
    "args": {
      "External id": 2487937,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4736
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657740473.604, "dur": 0.462,
    "args": {
      "External id": 2487938,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4737
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657740475.587, "dur": 0.327,
    "args": {
      "External id": 2487939,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4738
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657740477.732, "dur": 0.357,
    "args": {
      "External id": 2487940,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4739
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657740479.651, "dur": 2.419,
    "args": {
      "External id": 2487941,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4740
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657740496.099, "dur": 49.357,
    "args": {
      "External id": 2487942,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 4741
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1381179,
    "ts": 1514657740579.984, "dur": 124.055,
    "args": {
      "External id": 2487943,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "1", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 4742
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657740590.571, "dur": 4.365,
    "args": {
      "External id": 2487944,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4743
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1381179,
    "ts": 1514657740600.214, "dur": 10.425,
    "args": {
      "External id": 2487945,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 4744
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1381179,
    "ts": 1514657740604.363, "dur": 5.883,
    "args": {
      "External id": 2487946,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "12846080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 4745
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657740608.162, "dur": 0.574,
    "args": {
      "External id": 2487947,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 4746
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1381179,
    "ts": 1514657740618.054, "dur": 30.859,
    "args": {
      "External id": 2487948,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 4747
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657740620.250, "dur": 0.601,
    "args": {
      "External id": 2487949,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4748
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657740622.917, "dur": 0.364,
    "args": {
      "External id": 2487950,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6423296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4749
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657740627.386, "dur": 0.613,
    "args": {
      "External id": 2487951,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6947584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4750
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657740629.366, "dur": 0.631,
    "args": {
      "External id": 2487952,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7471872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4751
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657740631.478, "dur": 0.564,
    "args": {
      "External id": 2487953,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7996160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4752
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657740633.184, "dur": 2.847,
    "args": {
      "External id": 2487954,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8520448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4753
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657740637.488, "dur": 0.428,
    "args": {
      "External id": 2487955,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "8520704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4754
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657740639.205, "dur": 0.352,
    "args": {
      "External id": 2487956,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "9962496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4755
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657740641.402, "dur": 0.446,
    "args": {
      "External id": 2487957,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11404288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4756
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657740668.395, "dur": 28.149,
    "args": {
      "External id": 2487958,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 4757
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657740749.971, "dur": 107.593,
    "args": {
      "External id": 2487959,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 4758
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657740773.999, "dur": 80.259,
    "args": {
      "External id": 2487960,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4759, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514657740783.601, "dur": 66.487,
    "args": {
      "External id": 2487961,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 4760
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657740871.708, "dur": 1.731,
    "args": {
      "External id": 2487962,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4761, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657740947.860, "dur": 1712.873,
    "args": {
      "External id": 2487963,"Sequence number": 24740434, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4762
    }
  },
  {
    "ph": "f", "id": 214, "pid": 1336754, "tid": 1381179, "ts": 1514657740947.860,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657741099.495, "dur": 107.901,
    "args": {
      "External id": 2487964,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 4763
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514657741248.752, "dur": 42.904,
    "args": {
      "External id": 2487965,"kernel_hash": "cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/fu/cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 4764
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657741308.657, "dur": 50.664,
    "args": {
      "External id": 2487966,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 4765
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657741369.033, "dur": 32.512,
    "args": {
      "External id": 2487967,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4766
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657741407.839, "dur": 45.863,
    "args": {
      "External id": 2487968,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4767
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657741462.313, "dur": 27.607,
    "args": {
      "External id": 2487969,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4768
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657741497.486, "dur": 61.474,
    "args": {
      "External id": 2487970,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4769
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657741587.001, "dur": 26.665,
    "args": {
      "External id": 2487971,"kernel_hash": "cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/wm/cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 4770
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514657741633.244, "dur": 32.625,
    "args": {
      "External id": 2487972,"kernel_hash": "cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/br/cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4771
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657741684.744, "dur": 19.647,
    "args": {
      "External id": 2487973,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4772
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514657741720.567, "dur": 16.491,
    "args": {
      "External id": 2487974,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4773
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657741745.737, "dur": 32.977,
    "args": {
      "External id": 2487975,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4774
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657741781.836, "dur": 33.312,
    "args": {
      "External id": 2487976,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4775
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657741846.375, "dur": 218.886,
    "args": {
      "External id": 2487977,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 4776
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657741921.401, "dur": 8.977,
    "args": {
      "External id": 2487978,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4777
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657741932.064, "dur": 2.125,
    "args": {
      "External id": 2487979,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4778
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657742104.580, "dur": 28.335,
    "args": {
      "External id": 2487980,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4779
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657742144.499, "dur": 15.707,
    "args": {
      "External id": 2487981,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4780
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657742169.303, "dur": 43.022,
    "args": {
      "External id": 2487982,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4781
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657742217.777, "dur": 37.598,
    "args": {
      "External id": 2487983,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4782
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657742261.801, "dur": 22.330,
    "args": {
      "External id": 2487984,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4783
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657742290.551, "dur": 43.530,
    "args": {
      "External id": 2487985,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4784
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657742346.617, "dur": 26.199,
    "args": {
      "External id": 2487986,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4785
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657742380.569, "dur": 35.698,
    "args": {
      "External id": 2487987,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4786
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336754, "tid": 1381179,
    "ts": 1514657742436.969, "dur": 25.655,
    "args": {
      "External id": 2487988,"kernel_hash": "c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/6w/c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 4787
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657742481.640, "dur": 28.743,
    "args": {
      "External id": 2487989,"kernel_hash": "c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/4k/c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4788
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657742525.110, "dur": 33.648,
    "args": {
      "External id": 2487990,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4789
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514657742582.227, "dur": 16.800,
    "args": {
      "External id": 2487991,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4790
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336754, "tid": 1381179,
    "ts": 1514657742612.968, "dur": 20.729,
    "args": {
      "External id": 2487992,"kernel_hash": "cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/eb/cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 4791
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657742708.045, "dur": 14.608,
    "args": {
      "External id": 2487993,"Record function id": 0, "Ev Idx": 4792
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657742711.382, "dur": 10.385,
    "args": {
      "External id": 2487994,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4793
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657742715.535, "dur": 5.305,
    "args": {
      "External id": 2487995,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4794
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657742716.796, "dur": 3.923,
    "args": {
      "External id": 2487996,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4795
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657742726.555, "dur": 4.237,
    "args": {
      "External id": 2487997,"Record function id": 0, "Ev Idx": 4796
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657742728.068, "dur": 2.323,
    "args": {
      "External id": 2487998,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4797
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657742728.706, "dur": 1.246,
    "args": {
      "External id": 2487999,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4798
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657742729.028, "dur": 0.843,
    "args": {
      "External id": 2488000,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4799
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657742734.085, "dur": 3.947,
    "args": {
      "External id": 2488001,"Record function id": 0, "Ev Idx": 4800
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657742735.413, "dur": 2.229,
    "args": {
      "External id": 2488002,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4801
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657742735.910, "dur": 1.109,
    "args": {
      "External id": 2488003,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4802
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657742736.391, "dur": 0.549,
    "args": {
      "External id": 2488004,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4803
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657742744.220, "dur": 4.226,
    "args": {
      "External id": 2488005,"Record function id": 0, "Ev Idx": 4804
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657742745.610, "dur": 2.425,
    "args": {
      "External id": 2488006,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4805
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657742746.352, "dur": 1.280,
    "args": {
      "External id": 2488007,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4806
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657742746.777, "dur": 0.789,
    "args": {
      "External id": 2488008,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4807
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657742751.525, "dur": 6.306,
    "args": {
      "External id": 2488009,"Record function id": 0, "Ev Idx": 4808
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657742752.758, "dur": 4.645,
    "args": {
      "External id": 2488010,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4809
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657742753.715, "dur": 3.298,
    "args": {
      "External id": 2488011,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4810
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657742753.999, "dur": 2.946,
    "args": {
      "External id": 2488012,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4811
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657742760.889, "dur": 4.706,
    "args": {
      "External id": 2488013,"Record function id": 0, "Ev Idx": 4812
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657742762.211, "dur": 2.926,
    "args": {
      "External id": 2488014,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4813
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657742762.671, "dur": 2.066,
    "args": {
      "External id": 2488015,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4814
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657742763.882, "dur": 0.713,
    "args": {
      "External id": 2488016,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4815
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657742768.833, "dur": 3.922,
    "args": {
      "External id": 2488017,"Record function id": 0, "Ev Idx": 4816
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657742770.231, "dur": 2.134,
    "args": {
      "External id": 2488018,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4817
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657742770.795, "dur": 1.157,
    "args": {
      "External id": 2488019,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4818
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657742771.247, "dur": 0.630,
    "args": {
      "External id": 2488020,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4819
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657742775.841, "dur": 4.062,
    "args": {
      "External id": 2488021,"Record function id": 0, "Ev Idx": 4820
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657742777.301, "dur": 2.208,
    "args": {
      "External id": 2488022,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4821
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657742777.762, "dur": 1.126,
    "args": {
      "External id": 2488023,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4822
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657742778.166, "dur": 0.661,
    "args": {
      "External id": 2488024,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4823
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657742782.856, "dur": 4.117,
    "args": {
      "External id": 2488025,"Record function id": 0, "Ev Idx": 4824
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657742784.244, "dur": 2.342,
    "args": {
      "External id": 2488026,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4825
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657742784.717, "dur": 1.177,
    "args": {
      "External id": 2488027,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4826
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657742785.070, "dur": 0.756,
    "args": {
      "External id": 2488028,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4827
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657742790.450, "dur": 35468.891,
    "args": {
      "External id": 2488029,"Record function id": 0, "Sequence number": 24740433, "Fwd thread id": 1, "Ev Idx": 4828
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657742791.775, "dur": 35457.883,
    "args": {
      "External id": 2488030,"Sequence number": 24740433, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4829
    }
  },
  {
    "ph": "f", "id": 215, "pid": 1336754, "tid": 1381179, "ts": 1514657742791.775,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.6)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657742824.663, "dur": 39.908,
    "args": {
      "External id": 2488031,"Record function id": 0, "Ev Idx": 4830
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.6)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657742871.454, "dur": 69.067,
    "args": {
      "External id": 2488032,"Record function id": 0, "Ev Idx": 4831
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.6)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657742945.935, "dur": 35295.910,
    "args": {
      "External id": 2488033,"Record function id": 0, "Ev Idx": 4832
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657743087.582, "dur": 10.729,
    "args": {
      "External id": 2488034,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4833
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657743108.837, "dur": 4.867,
    "args": {
      "External id": 2488035,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 4834
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514657743128.399, "dur": 34307.330,
    "args": {
      "External id": 2488036,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4835
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514657743141.512, "dur": 34285.531,
    "args": {
      "External id": 2488037,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4836
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657743187.940, "dur": 14.599,
    "args": {
      "External id": 2488038,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4837
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514657743212.261, "dur": 34174.538,
    "args": {
      "External id": 2488039,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 4838
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657743214.821, "dur": 34171.304,
    "args": {
      "External id": 2488040,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 4839
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657743218.891, "dur": 5.337,
    "args": {
      "External id": 2488041,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4840
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657743225.711, "dur": 34156.979,
    "args": {
      "External id": 2488042,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 4841
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657777540.022, "dur": 10.595,
    "args": {
      "External id": 2488043,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 4842
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657777543.547, "dur": 6.514,
    "args": {
      "External id": 2488044,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4843
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657777580.019, "dur": 339.597,
    "args": {
      "External id": 2488045,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 4844
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657777605.721, "dur": 309.711,
    "args": {
      "External id": 2488046,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4845, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514657777616.952, "dur": 293.338,
    "args": {
      "External id": 2488047,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 4846
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657777939.895, "dur": 2.347,
    "args": {
      "External id": 2488048,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4847, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657778033.800, "dur": 8.479,
    "args": {
      "External id": 2488049,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4848
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657778086.561, "dur": 1.445,
    "args": {
      "External id": 2488050,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4849
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657778103.589, "dur": 1.328,
    "args": {
      "External id": 2488051,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4850
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657778117.036, "dur": 1.204,
    "args": {
      "External id": 2488052,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4851
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657778130.136, "dur": 2.974,
    "args": {
      "External id": 2488053,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4852
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657778143.592, "dur": 0.795,
    "args": {
      "External id": 2488054,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4853
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657778154.206, "dur": 1.043,
    "args": {
      "External id": 2488055,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4854
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657778165.854, "dur": 1.225,
    "args": {
      "External id": 2488056,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4855
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657778178.602, "dur": 2.781,
    "args": {
      "External id": 2488057,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4856
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657778273.956, "dur": 2834.629,
    "args": {
      "External id": 2488058,"Record function id": 0, "Ev Idx": 4857
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.5)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657778293.847, "dur": 1037.433,
    "args": {
      "External id": 2488059,"Record function id": 0, "Ev Idx": 4858
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.5)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657778310.829, "dur": 335.031,
    "args": {
      "External id": 2488060,"Record function id": 0, "Ev Idx": 4859
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657778392.044, "dur": 4.039,
    "args": {
      "External id": 2488061,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 4860
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657778399.382, "dur": 1.416,
    "args": {
      "External id": 2488062,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 4861
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657778402.587, "dur": 1.362,
    "args": {
      "External id": 2488063,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4862
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657778405.749, "dur": 1.412,
    "args": {
      "External id": 2488064,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4863
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657778410.284, "dur": 1.071,
    "args": {
      "External id": 2488065,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4864
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657778412.690, "dur": 1.209,
    "args": {
      "External id": 2488066,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4865
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657778415.967, "dur": 0.733,
    "args": {
      "External id": 2488067,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 4866
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657778418.111, "dur": 3.228,
    "args": {
      "External id": 2488068,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4867
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657778424.242, "dur": 1.510,
    "args": {
      "External id": 2488069,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4868
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657778427.258, "dur": 1.350,
    "args": {
      "External id": 2488070,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4869
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657778446.251, "dur": 168.090,
    "args": {
      "External id": 2488071,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 4870
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657778461.540, "dur": 148.143,
    "args": {
      "External id": 2488072,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 4871
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657778480.290, "dur": 13.118,
    "args": {
      "External id": 2488073,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4872
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514657778496.297, "dur": 84.147,
    "args": {
      "External id": 2488074,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 4873
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657778498.613, "dur": 81.436,
    "args": {
      "External id": 2488075,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 4874
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657778502.461, "dur": 6.255,
    "args": {
      "External id": 2488076,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4875
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657778511.983, "dur": 67.183,
    "args": {
      "External id": 2488077,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 4876
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.4", "pid": 1336754, "tid": 1381179,
    "ts": 1514657778739.028, "dur": 584.808,
    "args": {
      "External id": 2488078,"Record function id": 0, "Ev Idx": 4877
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.4)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657778754.920, "dur": 556.836,
    "args": {
      "External id": 2488079,"Record function id": 0, "Ev Idx": 4878
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657778817.401, "dur": 5.901,
    "args": {
      "External id": 2488080,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4879
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1381179,
    "ts": 1514657778842.514, "dur": 29.125,
    "args": {
      "External id": 2488081,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 4880
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657778847.643, "dur": 1.943,
    "args": {
      "External id": 2488082,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4881
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657778851.894, "dur": 0.521,
    "args": {
      "External id": 2488083,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4882
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657778853.904, "dur": 2.834,
    "args": {
      "External id": 2488084,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4883
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657778858.272, "dur": 0.463,
    "args": {
      "External id": 2488085,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4884
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657778859.983, "dur": 0.412,
    "args": {
      "External id": 2488086,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4885
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657778862.540, "dur": 0.486,
    "args": {
      "External id": 2488087,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4886
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657778864.322, "dur": 0.482,
    "args": {
      "External id": 2488088,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4887
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657778866.018, "dur": 0.336,
    "args": {
      "External id": 2488089,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4888
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657778868.156, "dur": 0.632,
    "args": {
      "External id": 2488090,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4889
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657778882.695, "dur": 31.304,
    "args": {
      "External id": 2488091,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 4890
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1381179,
    "ts": 1514657778943.910, "dur": 145.447,
    "args": {
      "External id": 2488092,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "1", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 4891
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657778953.727, "dur": 3.404,
    "args": {
      "External id": 2488093,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4892
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1381179,
    "ts": 1514657778962.172, "dur": 12.063,
    "args": {
      "External id": 2488094,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 4893
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1381179,
    "ts": 1514657778965.954, "dur": 7.861,
    "args": {
      "External id": 2488095,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "12846080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 4894
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657778969.645, "dur": 3.083,
    "args": {
      "External id": 2488096,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 4895
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1381179,
    "ts": 1514657779016.094, "dur": 30.011,
    "args": {
      "External id": 2488097,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 4896
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657779019.330, "dur": 0.960,
    "args": {
      "External id": 2488098,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4897
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657779022.485, "dur": 0.435,
    "args": {
      "External id": 2488099,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6423296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4898
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657779024.673, "dur": 0.562,
    "args": {
      "External id": 2488100,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6947584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4899
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657779026.435, "dur": 0.515,
    "args": {
      "External id": 2488101,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7471872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4900
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657779028.450, "dur": 0.387,
    "args": {
      "External id": 2488102,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7996160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4901
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657779030.158, "dur": 0.610,
    "args": {
      "External id": 2488103,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8520448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4902
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657779032.354, "dur": 0.564,
    "args": {
      "External id": 2488104,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "8520704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4903
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657779034.424, "dur": 2.299,
    "args": {
      "External id": 2488105,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "9962496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4904
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657779038.477, "dur": 0.264,
    "args": {
      "External id": 2488106,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11404288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4905
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657779058.780, "dur": 22.950,
    "args": {
      "External id": 2488107,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 4906
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657779134.646, "dur": 112.262,
    "args": {
      "External id": 2488108,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 4907
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657779160.848, "dur": 82.872,
    "args": {
      "External id": 2488109,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4908, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514657779170.131, "dur": 69.810,
    "args": {
      "External id": 2488110,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 4909
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657779262.048, "dur": 2.120,
    "args": {
      "External id": 2488111,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4910, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657779338.394, "dur": 1747.907,
    "args": {
      "External id": 2488112,"Sequence number": 24740432, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4911
    }
  },
  {
    "ph": "f", "id": 216, "pid": 1336754, "tid": 1381179, "ts": 1514657779338.394,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657779448.791, "dur": 123.716,
    "args": {
      "External id": 2488113,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 4912
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514657779615.623, "dur": 42.416,
    "args": {
      "External id": 2488114,"kernel_hash": "cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/fu/cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 4913
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657779677.300, "dur": 55.694,
    "args": {
      "External id": 2488115,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 4914
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657779742.912, "dur": 33.599,
    "args": {
      "External id": 2488116,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4915
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657779783.119, "dur": 51.937,
    "args": {
      "External id": 2488117,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4916
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657779842.563, "dur": 28.101,
    "args": {
      "External id": 2488118,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4917
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657779880.852, "dur": 42.736,
    "args": {
      "External id": 2488119,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4918
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657779947.920, "dur": 23.211,
    "args": {
      "External id": 2488120,"kernel_hash": "cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/wm/cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 4919
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514657780043.789, "dur": 37.845,
    "args": {
      "External id": 2488121,"kernel_hash": "cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/br/cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4920
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657780108.020, "dur": 22.424,
    "args": {
      "External id": 2488122,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4921
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514657780143.341, "dur": 16.701,
    "args": {
      "External id": 2488123,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4922
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657780173.249, "dur": 37.451,
    "args": {
      "External id": 2488124,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4923
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657780214.046, "dur": 34.996,
    "args": {
      "External id": 2488125,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4924
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657780276.891, "dur": 169.938,
    "args": {
      "External id": 2488126,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 4925
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657780356.086, "dur": 6.236,
    "args": {
      "External id": 2488127,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4926
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657780364.095, "dur": 2.712,
    "args": {
      "External id": 2488128,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4927
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657780482.060, "dur": 25.124,
    "args": {
      "External id": 2488129,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4928
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657780526.745, "dur": 33.862,
    "args": {
      "External id": 2488130,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4929
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657780571.346, "dur": 38.660,
    "args": {
      "External id": 2488131,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4930
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657780617.740, "dur": 35.154,
    "args": {
      "External id": 2488132,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4931
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657780675.031, "dur": 30.543,
    "args": {
      "External id": 2488133,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4932
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657780711.041, "dur": 33.808,
    "args": {
      "External id": 2488134,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4933
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657780750.945, "dur": 20.564,
    "args": {
      "External id": 2488135,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4934
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657780784.123, "dur": 29.597,
    "args": {
      "External id": 2488136,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4935
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336754, "tid": 1381179,
    "ts": 1514657780840.714, "dur": 23.993,
    "args": {
      "External id": 2488137,"kernel_hash": "c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/6w/c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 4936
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657780883.893, "dur": 27.293,
    "args": {
      "External id": 2488138,"kernel_hash": "c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/4k/c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4937
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657780926.474, "dur": 17.077,
    "args": {
      "External id": 2488139,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4938
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514657780958.707, "dur": 15.372,
    "args": {
      "External id": 2488140,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4939
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336754, "tid": 1381179,
    "ts": 1514657781034.937, "dur": 20.936,
    "args": {
      "External id": 2488141,"kernel_hash": "cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/eb/cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 4940
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657781130.768, "dur": 18.537,
    "args": {
      "External id": 2488142,"Record function id": 0, "Ev Idx": 4941
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657781134.391, "dur": 13.878,
    "args": {
      "External id": 2488143,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4942
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657781138.328, "dur": 8.788,
    "args": {
      "External id": 2488144,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4943
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657781139.901, "dur": 4.110,
    "args": {
      "External id": 2488145,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4944
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657781153.207, "dur": 7.433,
    "args": {
      "External id": 2488146,"Record function id": 0, "Ev Idx": 4945
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657781154.776, "dur": 5.409,
    "args": {
      "External id": 2488147,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4946
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657781155.858, "dur": 3.765,
    "args": {
      "External id": 2488148,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4947
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657781156.454, "dur": 3.082,
    "args": {
      "External id": 2488149,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4948
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657781164.049, "dur": 4.869,
    "args": {
      "External id": 2488150,"Record function id": 0, "Ev Idx": 4949
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657781165.553, "dur": 2.941,
    "args": {
      "External id": 2488151,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4950
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657781166.557, "dur": 1.508,
    "args": {
      "External id": 2488152,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4951
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657781167.182, "dur": 0.776,
    "args": {
      "External id": 2488153,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4952
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657781172.269, "dur": 4.704,
    "args": {
      "External id": 2488154,"Record function id": 0, "Ev Idx": 4953
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657781173.743, "dur": 2.836,
    "args": {
      "External id": 2488155,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4954
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657781174.588, "dur": 1.288,
    "args": {
      "External id": 2488156,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4955
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657781174.882, "dur": 0.891,
    "args": {
      "External id": 2488157,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4956
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657781180.082, "dur": 3.518,
    "args": {
      "External id": 2488158,"Record function id": 0, "Ev Idx": 4957
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657781181.426, "dur": 1.783,
    "args": {
      "External id": 2488159,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4958
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657781181.860, "dur": 0.935,
    "args": {
      "External id": 2488160,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4959
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657781182.114, "dur": 0.606,
    "args": {
      "External id": 2488161,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4960
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657781186.671, "dur": 3.255,
    "args": {
      "External id": 2488162,"Record function id": 0, "Ev Idx": 4961
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657781187.564, "dur": 1.978,
    "args": {
      "External id": 2488163,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4962
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657781188.017, "dur": 1.079,
    "args": {
      "External id": 2488164,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4963
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657781188.478, "dur": 0.553,
    "args": {
      "External id": 2488165,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4964
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657781193.158, "dur": 3.453,
    "args": {
      "External id": 2488166,"Record function id": 0, "Ev Idx": 4965
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657781194.142, "dur": 2.067,
    "args": {
      "External id": 2488167,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4966
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657781194.600, "dur": 0.966,
    "args": {
      "External id": 2488168,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4967
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657781194.889, "dur": 0.610,
    "args": {
      "External id": 2488169,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4968
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657781199.708, "dur": 6.814,
    "args": {
      "External id": 2488170,"Record function id": 0, "Ev Idx": 4969
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657781200.829, "dur": 5.254,
    "args": {
      "External id": 2488171,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4970
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657781201.489, "dur": 4.175,
    "args": {
      "External id": 2488172,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4971
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657781201.787, "dur": 3.811,
    "args": {
      "External id": 2488173,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4972
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657781209.547, "dur": 6.864,
    "args": {
      "External id": 2488174,"Record function id": 0, "Ev Idx": 4973
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657781210.592, "dur": 5.400,
    "args": {
      "External id": 2488175,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4974
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657781211.023, "dur": 4.253,
    "args": {
      "External id": 2488176,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4975
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657781212.885, "dur": 2.268,
    "args": {
      "External id": 2488177,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4976
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657781220.307, "dur": 38263.933,
    "args": {
      "External id": 2488178,"Record function id": 0, "Sequence number": 24740431, "Fwd thread id": 1, "Ev Idx": 4977
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657781221.606, "dur": 38253.716,
    "args": {
      "External id": 2488179,"Sequence number": 24740431, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4978
    }
  },
  {
    "ph": "f", "id": 217, "pid": 1336754, "tid": 1381179, "ts": 1514657781221.606,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.5)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657781250.913, "dur": 37.221,
    "args": {
      "External id": 2488180,"Record function id": 0, "Ev Idx": 4979
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.5)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657781295.657, "dur": 67.144,
    "args": {
      "External id": 2488181,"Record function id": 0, "Ev Idx": 4980
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.5)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657781367.986, "dur": 38099.175,
    "args": {
      "External id": 2488182,"Record function id": 0, "Ev Idx": 4981
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657781454.804, "dur": 7.296,
    "args": {
      "External id": 2488183,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4982
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657781471.447, "dur": 4.450,
    "args": {
      "External id": 2488184,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 4983
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514657781490.708, "dur": 37132.334,
    "args": {
      "External id": 2488185,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4984
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514657781503.448, "dur": 37111.258,
    "args": {
      "External id": 2488186,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4985
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657781566.025, "dur": 15.230,
    "args": {
      "External id": 2488187,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4986
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514657781590.047, "dur": 36982.922,
    "args": {
      "External id": 2488188,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 4987
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657781592.879, "dur": 36979.435,
    "args": {
      "External id": 2488189,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 4988
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657781596.775, "dur": 9.162,
    "args": {
      "External id": 2488190,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4989
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657781607.545, "dur": 36961.331,
    "args": {
      "External id": 2488191,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 4990
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657818709.424, "dur": 8.980,
    "args": {
      "External id": 2488192,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 4991
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657818712.366, "dur": 5.634,
    "args": {
      "External id": 2488193,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4992
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657818746.640, "dur": 425.910,
    "args": {
      "External id": 2488194,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 4993
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657818777.494, "dur": 389.688,
    "args": {
      "External id": 2488195,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4994, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514657818788.508, "dur": 372.282,
    "args": {
      "External id": 2488196,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 4995
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657819196.614, "dur": 2.617,
    "args": {
      "External id": 2488197,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4996, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657819261.782, "dur": 6.793,
    "args": {
      "External id": 2488198,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4997
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657819312.429, "dur": 1.142,
    "args": {
      "External id": 2488199,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4998
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657819331.365, "dur": 1.333,
    "args": {
      "External id": 2488200,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4999
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657819344.788, "dur": 0.872,
    "args": {
      "External id": 2488201,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5000
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657819356.736, "dur": 0.992,
    "args": {
      "External id": 2488202,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5001
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657819367.887, "dur": 0.905,
    "args": {
      "External id": 2488203,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5002
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657819383.060, "dur": 0.870,
    "args": {
      "External id": 2488204,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5003
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657819395.361, "dur": 1.701,
    "args": {
      "External id": 2488205,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5004
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657819407.297, "dur": 1.013,
    "args": {
      "External id": 2488206,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5005
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657819497.654, "dur": 2807.519,
    "args": {
      "External id": 2488207,"Record function id": 0, "Ev Idx": 5006
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.4)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657819516.612, "dur": 1073.357,
    "args": {
      "External id": 2488208,"Record function id": 0, "Ev Idx": 5007
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.4)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657819549.721, "dur": 322.387,
    "args": {
      "External id": 2488209,"Record function id": 0, "Ev Idx": 5008
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657819633.687, "dur": 4.988,
    "args": {
      "External id": 2488210,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 5009
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657819642.050, "dur": 1.076,
    "args": {
      "External id": 2488211,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5010
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657819646.277, "dur": 2.987,
    "args": {
      "External id": 2488212,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5011
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657819651.044, "dur": 0.981,
    "args": {
      "External id": 2488213,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5012
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657819653.871, "dur": 0.770,
    "args": {
      "External id": 2488214,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5013
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657819656.060, "dur": 0.990,
    "args": {
      "External id": 2488215,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5014
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657819659.620, "dur": 1.107,
    "args": {
      "External id": 2488216,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5015
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657819662.316, "dur": 0.731,
    "args": {
      "External id": 2488217,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5016
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657819664.863, "dur": 1.566,
    "args": {
      "External id": 2488218,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5017
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657819667.859, "dur": 1.287,
    "args": {
      "External id": 2488219,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5018
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657819687.326, "dur": 156.619,
    "args": {
      "External id": 2488220,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 5019
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657819703.932, "dur": 135.390,
    "args": {
      "External id": 2488221,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 5020
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657819722.844, "dur": 16.277,
    "args": {
      "External id": 2488222,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5021
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514657819742.134, "dur": 68.677,
    "args": {
      "External id": 2488223,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 5022
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657819744.964, "dur": 65.510,
    "args": {
      "External id": 2488224,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 5023
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657819749.365, "dur": 6.146,
    "args": {
      "External id": 2488225,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5024
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657819757.077, "dur": 52.880,
    "args": {
      "External id": 2488226,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 5025
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.3", "pid": 1336754, "tid": 1381179,
    "ts": 1514657819963.686, "dur": 619.028,
    "args": {
      "External id": 2488227,"Record function id": 0, "Ev Idx": 5026
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.3)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657819979.162, "dur": 590.230,
    "args": {
      "External id": 2488228,"Record function id": 0, "Ev Idx": 5027
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657820086.428, "dur": 6.274,
    "args": {
      "External id": 2488229,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5028
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1381179,
    "ts": 1514657820113.091, "dur": 29.963,
    "args": {
      "External id": 2488230,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 5029
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657820118.252, "dur": 1.650,
    "args": {
      "External id": 2488231,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5030
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657820122.988, "dur": 0.480,
    "args": {
      "External id": 2488232,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5031
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657820125.302, "dur": 0.450,
    "args": {
      "External id": 2488233,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5032
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657820127.038, "dur": 1.449,
    "args": {
      "External id": 2488234,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5033
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657820129.736, "dur": 0.410,
    "args": {
      "External id": 2488235,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5034
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657820131.325, "dur": 1.811,
    "args": {
      "External id": 2488236,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5035
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657820135.875, "dur": 0.372,
    "args": {
      "External id": 2488237,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5036
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657820137.514, "dur": 0.514,
    "args": {
      "External id": 2488238,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5037
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657820139.527, "dur": 0.338,
    "args": {
      "External id": 2488239,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5038
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657820155.082, "dur": 36.553,
    "args": {
      "External id": 2488240,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 5039
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1381179,
    "ts": 1514657820222.819, "dur": 101.160,
    "args": {
      "External id": 2488241,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "1", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 5040
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657820232.417, "dur": 3.662,
    "args": {
      "External id": 2488242,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5041
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1381179,
    "ts": 1514657820240.920, "dur": 10.461,
    "args": {
      "External id": 2488243,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 5042
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1381179,
    "ts": 1514657820244.809, "dur": 6.164,
    "args": {
      "External id": 2488244,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "12846080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 5043
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657820248.663, "dur": 1.205,
    "args": {
      "External id": 2488245,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 5044
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1381179,
    "ts": 1514657820258.288, "dur": 26.872,
    "args": {
      "External id": 2488246,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 5045
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657820260.502, "dur": 0.483,
    "args": {
      "External id": 2488247,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5046
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657820262.470, "dur": 0.691,
    "args": {
      "External id": 2488248,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6423296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5047
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657820264.848, "dur": 2.252,
    "args": {
      "External id": 2488249,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6947584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5048
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657820269.369, "dur": 0.415,
    "args": {
      "External id": 2488250,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7471872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5049
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657820270.924, "dur": 0.342,
    "args": {
      "External id": 2488251,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7996160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5050
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657820272.660, "dur": 0.316,
    "args": {
      "External id": 2488252,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8520448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5051
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657820274.267, "dur": 0.170,
    "args": {
      "External id": 2488253,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "8520704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5052
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657820276.062, "dur": 0.329,
    "args": {
      "External id": 2488254,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "9962496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5053
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657820277.844, "dur": 0.247,
    "args": {
      "External id": 2488255,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11404288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5054
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657820297.645, "dur": 19.189,
    "args": {
      "External id": 2488256,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 5055
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657820367.873, "dur": 116.482,
    "args": {
      "External id": 2488257,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 5056
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657820393.283, "dur": 87.677,
    "args": {
      "External id": 2488258,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 5057, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514657820402.661, "dur": 69.204,
    "args": {
      "External id": 2488259,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 5058
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657820499.685, "dur": 1.720,
    "args": {
      "External id": 2488260,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 5059, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657820597.133, "dur": 1686.090,
    "args": {
      "External id": 2488261,"Sequence number": 24740430, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 5060
    }
  },
  {
    "ph": "f", "id": 218, "pid": 1336754, "tid": 1381179, "ts": 1514657820597.133,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657820709.290, "dur": 106.797,
    "args": {
      "External id": 2488262,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 5061
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514657820857.106, "dur": 40.884,
    "args": {
      "External id": 2488263,"kernel_hash": "cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/fu/cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 5062
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657820914.779, "dur": 50.470,
    "args": {
      "External id": 2488264,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 5063
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657820975.080, "dur": 76.761,
    "args": {
      "External id": 2488265,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 5064
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657821064.376, "dur": 48.418,
    "args": {
      "External id": 2488266,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 5065
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657821120.067, "dur": 30.231,
    "args": {
      "External id": 2488267,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 5066
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657821158.353, "dur": 44.942,
    "args": {
      "External id": 2488268,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 5067
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657821229.606, "dur": 27.093,
    "args": {
      "External id": 2488269,"kernel_hash": "cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/wm/cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 5068
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514657821277.238, "dur": 29.974,
    "args": {
      "External id": 2488270,"kernel_hash": "cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/br/cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5069
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657821330.169, "dur": 20.243,
    "args": {
      "External id": 2488271,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 5070
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514657821366.662, "dur": 16.468,
    "args": {
      "External id": 2488272,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 5071
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657821391.735, "dur": 29.843,
    "args": {
      "External id": 2488273,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5072
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657821424.700, "dur": 33.133,
    "args": {
      "External id": 2488274,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5073
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657821487.870, "dur": 186.970,
    "args": {
      "External id": 2488275,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 5074
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657821582.343, "dur": 6.532,
    "args": {
      "External id": 2488276,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5075
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657821590.770, "dur": 2.948,
    "args": {
      "External id": 2488277,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5076
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657821711.185, "dur": 26.004,
    "args": {
      "External id": 2488278,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5077
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657821748.995, "dur": 15.953,
    "args": {
      "External id": 2488279,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5078
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657821773.018, "dur": 42.221,
    "args": {
      "External id": 2488280,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5079
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657821820.883, "dur": 36.980,
    "args": {
      "External id": 2488281,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5080
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657821864.221, "dur": 21.502,
    "args": {
      "External id": 2488282,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5081
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657821891.800, "dur": 35.761,
    "args": {
      "External id": 2488283,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5082
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657821933.148, "dur": 21.101,
    "args": {
      "External id": 2488284,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5083
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657821960.982, "dur": 80.275,
    "args": {
      "External id": 2488285,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5084
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336754, "tid": 1381179,
    "ts": 1514657822076.035, "dur": 24.812,
    "args": {
      "External id": 2488286,"kernel_hash": "c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/6w/c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 5085
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657822124.063, "dur": 24.964,
    "args": {
      "External id": 2488287,"kernel_hash": "c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/4k/c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5086
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657822170.611, "dur": 21.264,
    "args": {
      "External id": 2488288,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 5087
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514657822207.259, "dur": 16.906,
    "args": {
      "External id": 2488289,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 5088
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336754, "tid": 1381179,
    "ts": 1514657822237.464, "dur": 17.423,
    "args": {
      "External id": 2488290,"kernel_hash": "cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/eb/cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 5089
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657822327.080, "dur": 14.890,
    "args": {
      "External id": 2488291,"Record function id": 0, "Ev Idx": 5090
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657822330.474, "dur": 10.549,
    "args": {
      "External id": 2488292,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5091
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657822334.556, "dur": 5.426,
    "args": {
      "External id": 2488293,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5092
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657822336.146, "dur": 3.749,
    "args": {
      "External id": 2488294,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5093
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657822345.929, "dur": 4.760,
    "args": {
      "External id": 2488295,"Record function id": 0, "Ev Idx": 5094
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657822347.448, "dur": 2.796,
    "args": {
      "External id": 2488296,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5095
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657822348.297, "dur": 1.463,
    "args": {
      "External id": 2488297,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5096
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657822348.753, "dur": 0.930,
    "args": {
      "External id": 2488298,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5097
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657822353.997, "dur": 4.048,
    "args": {
      "External id": 2488299,"Record function id": 0, "Ev Idx": 5098
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657822355.137, "dur": 2.503,
    "args": {
      "External id": 2488300,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5099
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657822355.805, "dur": 1.402,
    "args": {
      "External id": 2488301,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5100
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657822356.500, "dur": 0.618,
    "args": {
      "External id": 2488302,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5101
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657822361.285, "dur": 4.624,
    "args": {
      "External id": 2488303,"Record function id": 0, "Ev Idx": 5102
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657822363.070, "dur": 2.415,
    "args": {
      "External id": 2488304,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5103
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657822363.765, "dur": 1.093,
    "args": {
      "External id": 2488305,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5104
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657822364.234, "dur": 0.550,
    "args": {
      "External id": 2488306,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5105
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657822369.138, "dur": 5.472,
    "args": {
      "External id": 2488307,"Record function id": 0, "Ev Idx": 5106
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657822370.256, "dur": 3.921,
    "args": {
      "External id": 2488308,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5107
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657822370.676, "dur": 2.943,
    "args": {
      "External id": 2488309,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5108
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657822371.151, "dur": 2.383,
    "args": {
      "External id": 2488310,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5109
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657822377.652, "dur": 5.128,
    "args": {
      "External id": 2488311,"Record function id": 0, "Ev Idx": 5110
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657822378.867, "dur": 3.513,
    "args": {
      "External id": 2488312,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5111
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657822379.310, "dur": 2.633,
    "args": {
      "External id": 2488313,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5112
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657822380.995, "dur": 0.882,
    "args": {
      "External id": 2488314,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5113
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657822386.255, "dur": 3.737,
    "args": {
      "External id": 2488315,"Record function id": 0, "Ev Idx": 5114
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657822387.528, "dur": 2.069,
    "args": {
      "External id": 2488316,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5115
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657822388.062, "dur": 1.082,
    "args": {
      "External id": 2488317,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5116
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657822388.531, "dur": 0.546,
    "args": {
      "External id": 2488318,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5117
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657822393.192, "dur": 30.175,
    "args": {
      "External id": 2488319,"Record function id": 0, "Ev Idx": 5118
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657822394.278, "dur": 28.652,
    "args": {
      "External id": 2488320,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5119
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657822417.627, "dur": 4.899,
    "args": {
      "External id": 2488321,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5120
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657822421.830, "dur": 0.624,
    "args": {
      "External id": 2488322,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5121
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657822426.574, "dur": 5.558,
    "args": {
      "External id": 2488323,"Record function id": 0, "Ev Idx": 5122
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657822428.287, "dur": 3.427,
    "args": {
      "External id": 2488324,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5123
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657822428.782, "dur": 2.381,
    "args": {
      "External id": 2488325,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5124
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657822430.487, "dur": 0.560,
    "args": {
      "External id": 2488326,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5125
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657822436.201, "dur": 35886.773,
    "args": {
      "External id": 2488327,"Record function id": 0, "Sequence number": 24740429, "Fwd thread id": 1, "Ev Idx": 5126
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657822437.888, "dur": 35874.630,
    "args": {
      "External id": 2488328,"Sequence number": 24740429, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 5127
    }
  },
  {
    "ph": "f", "id": 219, "pid": 1336754, "tid": 1381179, "ts": 1514657822437.888,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.4)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657822466.013, "dur": 42.809,
    "args": {
      "External id": 2488329,"Record function id": 0, "Ev Idx": 5128
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.4)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657822515.987, "dur": 86.931,
    "args": {
      "External id": 2488330,"Record function id": 0, "Ev Idx": 5129
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.4)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657822613.887, "dur": 35691.195,
    "args": {
      "External id": 2488331,"Record function id": 0, "Ev Idx": 5130
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657822707.397, "dur": 7.011,
    "args": {
      "External id": 2488332,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5131
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657822724.320, "dur": 4.934,
    "args": {
      "External id": 2488333,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 5132
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514657822744.096, "dur": 34735.276,
    "args": {
      "External id": 2488334,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 5133
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514657822756.719, "dur": 34713.807,
    "args": {
      "External id": 2488335,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 5134
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657822797.932, "dur": 14.402,
    "args": {
      "External id": 2488336,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5135
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514657822823.210, "dur": 34610.685,
    "args": {
      "External id": 2488337,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 5136
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657822825.564, "dur": 34607.610,
    "args": {
      "External id": 2488338,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 5137
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657822829.442, "dur": 6.640,
    "args": {
      "External id": 2488339,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5138
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657822837.650, "dur": 34592.071,
    "args": {
      "External id": 2488340,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 5139
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657857579.269, "dur": 8.979,
    "args": {
      "External id": 2488341,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 5140
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657857581.950, "dur": 5.790,
    "args": {
      "External id": 2488342,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5141
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657857616.811, "dur": 360.636,
    "args": {
      "External id": 2488343,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 5142
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657857645.028, "dur": 327.098,
    "args": {
      "External id": 2488344,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 5143, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514657857655.609, "dur": 310.906,
    "args": {
      "External id": 2488345,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 5144
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657858028.022, "dur": 3.649,
    "args": {
      "External id": 2488346,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 5145, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657858094.627, "dur": 6.787,
    "args": {
      "External id": 2488347,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5146
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657858144.812, "dur": 1.533,
    "args": {
      "External id": 2488348,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5147
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657858162.854, "dur": 1.644,
    "args": {
      "External id": 2488349,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5148
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657858176.909, "dur": 3.837,
    "args": {
      "External id": 2488350,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5149
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657858193.123, "dur": 0.940,
    "args": {
      "External id": 2488351,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5150
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657858204.078, "dur": 0.967,
    "args": {
      "External id": 2488352,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5151
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657858217.405, "dur": 1.174,
    "args": {
      "External id": 2488353,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5152
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657858229.904, "dur": 3.174,
    "args": {
      "External id": 2488354,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5153
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657858246.286, "dur": 0.901,
    "args": {
      "External id": 2488355,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5154
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657858337.184, "dur": 2793.149,
    "args": {
      "External id": 2488356,"Record function id": 0, "Ev Idx": 5155
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.3)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657858357.302, "dur": 1044.812,
    "args": {
      "External id": 2488357,"Record function id": 0, "Ev Idx": 5156
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.3)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657858370.272, "dur": 328.064,
    "args": {
      "External id": 2488358,"Record function id": 0, "Ev Idx": 5157
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657858450.099, "dur": 4.304,
    "args": {
      "External id": 2488359,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 5158
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657858458.093, "dur": 1.015,
    "args": {
      "External id": 2488360,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5159
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657858462.344, "dur": 0.917,
    "args": {
      "External id": 2488361,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5160
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657858465.026, "dur": 0.897,
    "args": {
      "External id": 2488362,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5161
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657858467.483, "dur": 0.993,
    "args": {
      "External id": 2488363,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5162
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657858470.005, "dur": 2.572,
    "args": {
      "External id": 2488364,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5163
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657858475.405, "dur": 0.843,
    "args": {
      "External id": 2488365,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5164
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657858477.707, "dur": 0.788,
    "args": {
      "External id": 2488366,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5165
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657858480.000, "dur": 1.205,
    "args": {
      "External id": 2488367,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5166
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657858482.803, "dur": 1.379,
    "args": {
      "External id": 2488368,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5167
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657858502.493, "dur": 165.809,
    "args": {
      "External id": 2488369,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 5168
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657858518.350, "dur": 145.349,
    "args": {
      "External id": 2488370,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 5169
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657858552.364, "dur": 12.540,
    "args": {
      "External id": 2488371,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5170
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514657858568.082, "dur": 66.801,
    "args": {
      "External id": 2488372,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 5171
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657858570.507, "dur": 64.027,
    "args": {
      "External id": 2488373,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 5172
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657858574.394, "dur": 6.424,
    "args": {
      "External id": 2488374,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5173
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657858582.256, "dur": 51.778,
    "args": {
      "External id": 2488375,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 5174
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657858794.803, "dur": 599.648,
    "args": {
      "External id": 2488376,"Record function id": 0, "Ev Idx": 5175
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.2)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657858812.273, "dur": 569.922,
    "args": {
      "External id": 2488377,"Record function id": 0, "Ev Idx": 5176
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657858875.225, "dur": 5.148,
    "args": {
      "External id": 2488378,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5177
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1381179,
    "ts": 1514657858900.436, "dur": 32.431,
    "args": {
      "External id": 2488379,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 5178
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657858905.749, "dur": 5.269,
    "args": {
      "External id": 2488380,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5179
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657858912.813, "dur": 0.619,
    "args": {
      "External id": 2488381,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5180
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657858915.083, "dur": 0.537,
    "args": {
      "External id": 2488382,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5181
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657858917.829, "dur": 0.378,
    "args": {
      "External id": 2488383,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5182
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657858919.661, "dur": 0.342,
    "args": {
      "External id": 2488384,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5183
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657858921.148, "dur": 0.463,
    "args": {
      "External id": 2488385,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5184
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657858923.445, "dur": 0.440,
    "args": {
      "External id": 2488386,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5185
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657858925.509, "dur": 0.309,
    "args": {
      "External id": 2488387,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5186
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657858927.437, "dur": 2.841,
    "args": {
      "External id": 2488388,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5187
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657858945.240, "dur": 31.800,
    "args": {
      "External id": 2488389,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 5188
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1381179,
    "ts": 1514657859048.598, "dur": 111.702,
    "args": {
      "External id": 2488390,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "1", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 5189
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657859059.544, "dur": 5.007,
    "args": {
      "External id": 2488391,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5190
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1381179,
    "ts": 1514657859069.924, "dur": 10.702,
    "args": {
      "External id": 2488392,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 5191
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1381179,
    "ts": 1514657859073.974, "dur": 6.261,
    "args": {
      "External id": 2488393,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "12846080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 5192
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657859077.714, "dur": 0.731,
    "args": {
      "External id": 2488394,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 5193
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1381179,
    "ts": 1514657859087.783, "dur": 29.461,
    "args": {
      "External id": 2488395,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 5194
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657859089.974, "dur": 0.508,
    "args": {
      "External id": 2488396,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5195
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657859092.292, "dur": 0.550,
    "args": {
      "External id": 2488397,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6423296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5196
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657859094.175, "dur": 1.208,
    "args": {
      "External id": 2488398,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6947584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5197
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657859097.090, "dur": 0.559,
    "args": {
      "External id": 2488399,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7471872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5198
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657859099.186, "dur": 0.595,
    "args": {
      "External id": 2488400,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7996160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5199
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657859101.581, "dur": 2.516,
    "args": {
      "External id": 2488401,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8520448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5200
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657859105.425, "dur": 0.544,
    "args": {
      "External id": 2488402,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "8520704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5201
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657859107.724, "dur": 0.323,
    "args": {
      "External id": 2488403,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "9962496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5202
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657859109.526, "dur": 0.404,
    "args": {
      "External id": 2488404,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11404288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5203
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657859131.020, "dur": 22.283,
    "args": {
      "External id": 2488405,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 5204
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657859205.411, "dur": 112.063,
    "args": {
      "External id": 2488406,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 5205
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657859232.076, "dur": 82.140,
    "args": {
      "External id": 2488407,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 5206, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514657859241.711, "dur": 68.532,
    "args": {
      "External id": 2488408,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 5207
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657859332.320, "dur": 1.806,
    "args": {
      "External id": 2488409,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 5208, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657859409.039, "dur": 1698.988,
    "args": {
      "External id": 2488410,"Sequence number": 24740428, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 5209
    }
  },
  {
    "ph": "f", "id": 220, "pid": 1336754, "tid": 1381179, "ts": 1514657859409.039,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657859519.782, "dur": 123.888,
    "args": {
      "External id": 2488411,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 5210
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514657859688.114, "dur": 44.764,
    "args": {
      "External id": 2488412,"kernel_hash": "cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/fu/cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 5211
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657859750.369, "dur": 51.090,
    "args": {
      "External id": 2488413,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 5212
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657859811.412, "dur": 32.369,
    "args": {
      "External id": 2488414,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 5213
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657859850.370, "dur": 43.955,
    "args": {
      "External id": 2488415,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 5214
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657859903.415, "dur": 27.423,
    "args": {
      "External id": 2488416,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 5215
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657859938.921, "dur": 79.903,
    "args": {
      "External id": 2488417,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 5216
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657860048.575, "dur": 30.970,
    "args": {
      "External id": 2488418,"kernel_hash": "cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/wm/cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 5217
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514657860099.576, "dur": 35.056,
    "args": {
      "External id": 2488419,"kernel_hash": "cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/br/cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5218
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657860155.584, "dur": 20.198,
    "args": {
      "External id": 2488420,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 5219
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514657860190.216, "dur": 16.707,
    "args": {
      "External id": 2488421,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 5220
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657860217.182, "dur": 34.557,
    "args": {
      "External id": 2488422,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5221
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657860254.954, "dur": 32.864,
    "args": {
      "External id": 2488423,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5222
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657860317.317, "dur": 163.185,
    "args": {
      "External id": 2488424,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 5223
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657860392.198, "dur": 5.878,
    "args": {
      "External id": 2488425,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5224
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657860399.695, "dur": 2.542,
    "args": {
      "External id": 2488426,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5225
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657860515.271, "dur": 42.965,
    "args": {
      "External id": 2488427,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5226
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657860572.048, "dur": 17.031,
    "args": {
      "External id": 2488428,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5227
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657860599.953, "dur": 42.140,
    "args": {
      "External id": 2488429,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5228
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657860648.100, "dur": 40.064,
    "args": {
      "External id": 2488430,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5229
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657860694.698, "dur": 25.117,
    "args": {
      "External id": 2488431,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5230
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657860724.260, "dur": 29.888,
    "args": {
      "External id": 2488432,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5231
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657860761.335, "dur": 36.281,
    "args": {
      "External id": 2488433,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5232
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657860810.983, "dur": 34.826,
    "args": {
      "External id": 2488434,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5233
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336754, "tid": 1381179,
    "ts": 1514657860867.221, "dur": 22.694,
    "args": {
      "External id": 2488435,"kernel_hash": "c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/6w/c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 5234
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657860911.230, "dur": 25.966,
    "args": {
      "External id": 2488436,"kernel_hash": "c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/4k/c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5235
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657860955.910, "dur": 17.610,
    "args": {
      "External id": 2488437,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 5236
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514657861026.550, "dur": 19.305,
    "args": {
      "External id": 2488438,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 5237
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336754, "tid": 1381179,
    "ts": 1514657861060.081, "dur": 18.584,
    "args": {
      "External id": 2488439,"kernel_hash": "cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/eb/cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 5238
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657861152.150, "dur": 16.418,
    "args": {
      "External id": 2488440,"Record function id": 0, "Ev Idx": 5239
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657861155.388, "dur": 12.300,
    "args": {
      "External id": 2488441,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5240
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657861159.464, "dur": 7.309,
    "args": {
      "External id": 2488442,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5241
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657861160.833, "dur": 5.820,
    "args": {
      "External id": 2488443,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5242
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657861172.348, "dur": 5.925,
    "args": {
      "External id": 2488444,"Record function id": 0, "Ev Idx": 5243
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657861174.174, "dur": 3.628,
    "args": {
      "External id": 2488445,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5244
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657861175.348, "dur": 1.966,
    "args": {
      "External id": 2488446,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5245
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657861176.274, "dur": 0.948,
    "args": {
      "External id": 2488447,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5246
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657861181.813, "dur": 4.410,
    "args": {
      "External id": 2488448,"Record function id": 0, "Ev Idx": 5247
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657861183.248, "dur": 2.554,
    "args": {
      "External id": 2488449,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5248
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657861183.761, "dur": 1.487,
    "args": {
      "External id": 2488450,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5249
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657861184.317, "dur": 0.834,
    "args": {
      "External id": 2488451,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5250
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657861189.502, "dur": 3.942,
    "args": {
      "External id": 2488452,"Record function id": 0, "Ev Idx": 5251
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657861190.654, "dur": 2.401,
    "args": {
      "External id": 2488453,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5252
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657861191.367, "dur": 1.233,
    "args": {
      "External id": 2488454,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5253
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657861191.860, "dur": 0.646,
    "args": {
      "External id": 2488455,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5254
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657861199.654, "dur": 3.330,
    "args": {
      "External id": 2488456,"Record function id": 0, "Ev Idx": 5255
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657861200.624, "dur": 1.977,
    "args": {
      "External id": 2488457,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5256
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657861201.066, "dur": 1.106,
    "args": {
      "External id": 2488458,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5257
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657861201.526, "dur": 0.578,
    "args": {
      "External id": 2488459,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5258
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657861206.073, "dur": 3.611,
    "args": {
      "External id": 2488460,"Record function id": 0, "Ev Idx": 5259
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657861207.132, "dur": 2.145,
    "args": {
      "External id": 2488461,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5260
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657861207.726, "dur": 1.093,
    "args": {
      "External id": 2488462,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5261
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657861208.121, "dur": 0.633,
    "args": {
      "External id": 2488463,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5262
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657861212.855, "dur": 4.655,
    "args": {
      "External id": 2488464,"Record function id": 0, "Ev Idx": 5263
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657861213.896, "dur": 3.164,
    "args": {
      "External id": 2488465,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5264
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657861214.377, "dur": 2.256,
    "args": {
      "External id": 2488466,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5265
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657861215.992, "dur": 0.553,
    "args": {
      "External id": 2488467,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5266
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657861220.618, "dur": 9.728,
    "args": {
      "External id": 2488468,"Record function id": 0, "Ev Idx": 5267
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657861221.685, "dur": 8.269,
    "args": {
      "External id": 2488469,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5268
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657861226.200, "dur": 3.298,
    "args": {
      "External id": 2488470,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5269
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657861226.531, "dur": 2.899,
    "args": {
      "External id": 2488471,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5270
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657861233.403, "dur": 4.141,
    "args": {
      "External id": 2488472,"Record function id": 0, "Ev Idx": 5271
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657861234.586, "dur": 2.568,
    "args": {
      "External id": 2488473,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5272
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657861235.377, "dur": 1.353,
    "args": {
      "External id": 2488474,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5273
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657861235.638, "dur": 1.017,
    "args": {
      "External id": 2488475,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5274
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657861241.586, "dur": 36858.172,
    "args": {
      "External id": 2488476,"Record function id": 0, "Sequence number": 24740427, "Fwd thread id": 1, "Ev Idx": 5275
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657861243.057, "dur": 36847.668,
    "args": {
      "External id": 2488477,"Sequence number": 24740427, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 5276
    }
  },
  {
    "ph": "f", "id": 221, "pid": 1336754, "tid": 1381179, "ts": 1514657861243.057,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.3)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657861272.879, "dur": 37.235,
    "args": {
      "External id": 2488478,"Record function id": 0, "Ev Idx": 5277
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.3)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657861317.317, "dur": 67.266,
    "args": {
      "External id": 2488479,"Record function id": 0, "Ev Idx": 5278
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.3)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657861390.171, "dur": 36692.291,
    "args": {
      "External id": 2488480,"Record function id": 0, "Ev Idx": 5279
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657861475.188, "dur": 6.944,
    "args": {
      "External id": 2488481,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5280
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657861491.511, "dur": 4.344,
    "args": {
      "External id": 2488482,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 5281
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514657861510.044, "dur": 35718.172,
    "args": {
      "External id": 2488483,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 5282
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514657861526.899, "dur": 35692.147,
    "args": {
      "External id": 2488484,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 5283
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657861589.073, "dur": 14.760,
    "args": {
      "External id": 2488485,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5284
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514657861610.339, "dur": 35570.497,
    "args": {
      "External id": 2488486,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 5285
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657861612.739, "dur": 35567.455,
    "args": {
      "External id": 2488487,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 5286
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657861616.622, "dur": 6.967,
    "args": {
      "External id": 2488488,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5287
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657861626.574, "dur": 35549.953,
    "args": {
      "External id": 2488489,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 5288
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657897315.822, "dur": 8.968,
    "args": {
      "External id": 2488490,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 5289
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657897318.404, "dur": 5.989,
    "args": {
      "External id": 2488491,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5290
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657897355.254, "dur": 386.822,
    "args": {
      "External id": 2488492,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 5291
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657897383.796, "dur": 352.962,
    "args": {
      "External id": 2488493,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 5292, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514657897394.659, "dur": 336.062,
    "args": {
      "External id": 2488494,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 5293
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657897763.659, "dur": 2.395,
    "args": {
      "External id": 2488495,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 5294, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657897828.387, "dur": 6.956,
    "args": {
      "External id": 2488496,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5295
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657897878.588, "dur": 1.601,
    "args": {
      "External id": 2488497,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5296
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657897898.106, "dur": 3.627,
    "args": {
      "External id": 2488498,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5297
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657897914.497, "dur": 1.125,
    "args": {
      "External id": 2488499,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5298
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657897926.997, "dur": 1.115,
    "args": {
      "External id": 2488500,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5299
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657897938.801, "dur": 1.266,
    "args": {
      "External id": 2488501,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5300
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657897953.251, "dur": 3.431,
    "args": {
      "External id": 2488502,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5301
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657897971.425, "dur": 1.125,
    "args": {
      "External id": 2488503,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5302
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657898015.823, "dur": 1.456,
    "args": {
      "External id": 2488504,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5303
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657898115.214, "dur": 2747.406,
    "args": {
      "External id": 2488505,"Record function id": 0, "Ev Idx": 5304
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.2)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657898134.360, "dur": 1045.739,
    "args": {
      "External id": 2488506,"Record function id": 0, "Ev Idx": 5305
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.2)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657898147.890, "dur": 309.369,
    "args": {
      "External id": 2488507,"Record function id": 0, "Ev Idx": 5306
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657898231.117, "dur": 4.570,
    "args": {
      "External id": 2488508,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 5307
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657898238.733, "dur": 1.252,
    "args": {
      "External id": 2488509,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5308
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657898242.704, "dur": 0.995,
    "args": {
      "External id": 2488510,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5309
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657898245.622, "dur": 2.533,
    "args": {
      "External id": 2488511,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5310
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657898249.473, "dur": 1.319,
    "args": {
      "External id": 2488512,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5311
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657898252.065, "dur": 0.882,
    "args": {
      "External id": 2488513,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5312
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657898255.428, "dur": 0.924,
    "args": {
      "External id": 2488514,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5313
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657898257.941, "dur": 0.797,
    "args": {
      "External id": 2488515,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5314
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657898259.842, "dur": 1.389,
    "args": {
      "External id": 2488516,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5315
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657898262.609, "dur": 1.159,
    "args": {
      "External id": 2488517,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5316
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657898282.808, "dur": 148.331,
    "args": {
      "External id": 2488518,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 5317
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657898297.933, "dur": 128.619,
    "args": {
      "External id": 2488519,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 5318
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657898315.219, "dur": 12.472,
    "args": {
      "External id": 2488520,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5319
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514657898330.544, "dur": 67.740,
    "args": {
      "External id": 2488521,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 5320
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657898333.114, "dur": 64.860,
    "args": {
      "External id": 2488522,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 5321
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657898336.827, "dur": 8.039,
    "args": {
      "External id": 2488523,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5322
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657898346.417, "dur": 50.795,
    "args": {
      "External id": 2488524,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 5323
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657898572.791, "dur": 599.339,
    "args": {
      "External id": 2488525,"Record function id": 0, "Ev Idx": 5324
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.1)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657898590.972, "dur": 568.893,
    "args": {
      "External id": 2488526,"Record function id": 0, "Ev Idx": 5325
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657898659.121, "dur": 5.918,
    "args": {
      "External id": 2488527,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5326
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1381179,
    "ts": 1514657898691.266, "dur": 29.374,
    "args": {
      "External id": 2488528,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 5327
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657898699.311, "dur": 2.131,
    "args": {
      "External id": 2488529,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5328
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657898703.072, "dur": 0.556,
    "args": {
      "External id": 2488530,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5329
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657898704.513, "dur": 0.666,
    "args": {
      "External id": 2488531,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5330
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657898706.847, "dur": 0.456,
    "args": {
      "External id": 2488532,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5331
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657898708.294, "dur": 0.520,
    "args": {
      "External id": 2488533,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5332
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657898710.141, "dur": 0.545,
    "args": {
      "External id": 2488534,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5333
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657898711.972, "dur": 2.229,
    "args": {
      "External id": 2488535,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5334
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657898715.247, "dur": 0.495,
    "args": {
      "External id": 2488536,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5335
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657898716.965, "dur": 0.625,
    "args": {
      "External id": 2488537,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5336
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657898732.128, "dur": 34.440,
    "args": {
      "External id": 2488538,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 5337
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1381179,
    "ts": 1514657898797.255, "dur": 99.450,
    "args": {
      "External id": 2488539,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "1", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 5338
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657898807.472, "dur": 3.444,
    "args": {
      "External id": 2488540,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5339
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1381179,
    "ts": 1514657898816.248, "dur": 9.910,
    "args": {
      "External id": 2488541,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 5340
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1381179,
    "ts": 1514657898820.454, "dur": 5.319,
    "args": {
      "External id": 2488542,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "12846080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 5341
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657898823.867, "dur": 0.633,
    "args": {
      "External id": 2488543,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 5342
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1381179,
    "ts": 1514657898832.347, "dur": 25.153,
    "args": {
      "External id": 2488544,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 5343
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657898834.551, "dur": 0.267,
    "args": {
      "External id": 2488545,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5344
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657898835.848, "dur": 0.617,
    "args": {
      "External id": 2488546,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6423296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5345
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657898837.193, "dur": 0.849,
    "args": {
      "External id": 2488547,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6947584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5346
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657898839.372, "dur": 2.073,
    "args": {
      "External id": 2488548,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7471872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5347
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657898842.553, "dur": 0.426,
    "args": {
      "External id": 2488549,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7996160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5348
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657898844.305, "dur": 0.391,
    "args": {
      "External id": 2488550,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8520448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5349
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657898848.512, "dur": 0.302,
    "args": {
      "External id": 2488551,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "8520704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5350
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657898849.399, "dur": 0.367,
    "args": {
      "External id": 2488552,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "9962496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5351
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657898850.573, "dur": 0.296,
    "args": {
      "External id": 2488553,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11404288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5352
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657898869.020, "dur": 20.263,
    "args": {
      "External id": 2488554,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 5353
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657898939.560, "dur": 151.016,
    "args": {
      "External id": 2488555,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 5354
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657898963.333, "dur": 123.583,
    "args": {
      "External id": 2488556,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 5355, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514657898972.444, "dur": 109.840,
    "args": {
      "External id": 2488557,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 5356
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657899106.725, "dur": 2.078,
    "args": {
      "External id": 2488558,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 5357, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657899187.036, "dur": 1653.725,
    "args": {
      "External id": 2488559,"Sequence number": 24740426, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 5358
    }
  },
  {
    "ph": "f", "id": 222, "pid": 1336754, "tid": 1381179, "ts": 1514657899187.036,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657899296.829, "dur": 104.771,
    "args": {
      "External id": 2488560,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 5359
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514657899443.747, "dur": 40.619,
    "args": {
      "External id": 2488561,"kernel_hash": "cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/fu/cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 5360
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657899500.374, "dur": 74.713,
    "args": {
      "External id": 2488562,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 5361
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657899588.433, "dur": 36.420,
    "args": {
      "External id": 2488563,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 5362
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657899633.357, "dur": 46.491,
    "args": {
      "External id": 2488564,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 5363
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657899687.672, "dur": 26.583,
    "args": {
      "External id": 2488565,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 5364
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657899721.882, "dur": 41.985,
    "args": {
      "External id": 2488566,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 5365
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657899787.920, "dur": 25.964,
    "args": {
      "External id": 2488567,"kernel_hash": "cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/wm/cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 5366
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514657899832.437, "dur": 31.715,
    "args": {
      "External id": 2488568,"kernel_hash": "cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/br/cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5367
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657899884.715, "dur": 18.230,
    "args": {
      "External id": 2488569,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 5368
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514657899920.539, "dur": 16.924,
    "args": {
      "External id": 2488570,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 5369
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657899944.699, "dur": 29.460,
    "args": {
      "External id": 2488571,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5370
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657899977.376, "dur": 74.477,
    "args": {
      "External id": 2488572,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5371
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657900084.898, "dur": 175.182,
    "args": {
      "External id": 2488573,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 5372
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657900161.473, "dur": 7.968,
    "args": {
      "External id": 2488574,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5373
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657900171.216, "dur": 2.489,
    "args": {
      "External id": 2488575,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5374
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657900296.795, "dur": 29.762,
    "args": {
      "External id": 2488576,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5375
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657900337.686, "dur": 15.492,
    "args": {
      "External id": 2488577,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5376
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657900360.903, "dur": 38.047,
    "args": {
      "External id": 2488578,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5377
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657900404.444, "dur": 35.066,
    "args": {
      "External id": 2488579,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5378
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657900446.346, "dur": 21.777,
    "args": {
      "External id": 2488580,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5379
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657900489.917, "dur": 55.905,
    "args": {
      "External id": 2488581,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5380
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657900556.836, "dur": 25.156,
    "args": {
      "External id": 2488582,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5381
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657900588.786, "dur": 34.745,
    "args": {
      "External id": 2488583,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5382
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336754, "tid": 1381179,
    "ts": 1514657900645.526, "dur": 25.411,
    "args": {
      "External id": 2488584,"kernel_hash": "c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/6w/c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 5383
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657900690.211, "dur": 29.642,
    "args": {
      "External id": 2488585,"kernel_hash": "c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/4k/c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5384
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657900734.853, "dur": 17.969,
    "args": {
      "External id": 2488586,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 5385
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514657900767.406, "dur": 15.798,
    "args": {
      "External id": 2488587,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 5386
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336754, "tid": 1381179,
    "ts": 1514657900794.701, "dur": 17.529,
    "args": {
      "External id": 2488588,"kernel_hash": "cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/eb/cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 5387
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657900884.135, "dur": 15.263,
    "args": {
      "External id": 2488589,"Record function id": 0, "Ev Idx": 5388
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657900887.550, "dur": 10.915,
    "args": {
      "External id": 2488590,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5389
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657900891.688, "dur": 5.846,
    "args": {
      "External id": 2488591,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5390
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657900893.551, "dur": 3.852,
    "args": {
      "External id": 2488592,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5391
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657900903.187, "dur": 5.944,
    "args": {
      "External id": 2488593,"Record function id": 0, "Ev Idx": 5392
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657900904.884, "dur": 3.786,
    "args": {
      "External id": 2488594,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5393
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657900905.976, "dur": 2.179,
    "args": {
      "External id": 2488595,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5394
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657900907.027, "dur": 1.026,
    "args": {
      "External id": 2488596,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5395
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657900912.463, "dur": 5.290,
    "args": {
      "External id": 2488597,"Record function id": 0, "Ev Idx": 5396
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657900913.786, "dur": 3.545,
    "args": {
      "External id": 2488598,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5397
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657900914.415, "dur": 2.471,
    "args": {
      "External id": 2488599,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5398
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657900915.341, "dur": 1.476,
    "args": {
      "External id": 2488600,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5399
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657900920.936, "dur": 5.165,
    "args": {
      "External id": 2488601,"Record function id": 0, "Ev Idx": 5400
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657900922.840, "dur": 2.876,
    "args": {
      "External id": 2488602,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5401
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657900923.551, "dur": 1.720,
    "args": {
      "External id": 2488603,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5402
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657900924.101, "dur": 1.108,
    "args": {
      "External id": 2488604,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5403
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657900929.372, "dur": 4.780,
    "args": {
      "External id": 2488605,"Record function id": 0, "Ev Idx": 5404
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657900930.949, "dur": 2.779,
    "args": {
      "External id": 2488606,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5405
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657900931.996, "dur": 1.300,
    "args": {
      "External id": 2488607,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5406
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657900932.538, "dur": 0.692,
    "args": {
      "External id": 2488608,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5407
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657900937.267, "dur": 3.973,
    "args": {
      "External id": 2488609,"Record function id": 0, "Ev Idx": 5408
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657900938.558, "dur": 2.260,
    "args": {
      "External id": 2488610,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5409
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657900939.055, "dur": 1.192,
    "args": {
      "External id": 2488611,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5410
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657900939.562, "dur": 0.568,
    "args": {
      "External id": 2488612,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5411
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657900944.594, "dur": 6.088,
    "args": {
      "External id": 2488613,"Record function id": 0, "Ev Idx": 5412
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657900945.777, "dur": 4.517,
    "args": {
      "External id": 2488614,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5413
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657900946.476, "dur": 3.400,
    "args": {
      "External id": 2488615,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5414
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657900946.751, "dur": 3.052,
    "args": {
      "External id": 2488616,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5415
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657900953.809, "dur": 3.838,
    "args": {
      "External id": 2488617,"Record function id": 0, "Ev Idx": 5416
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657900955.060, "dur": 2.173,
    "args": {
      "External id": 2488618,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5417
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657900955.600, "dur": 1.189,
    "args": {
      "External id": 2488619,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5418
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657900955.878, "dur": 0.818,
    "args": {
      "External id": 2488620,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5419
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657900960.716, "dur": 4.739,
    "args": {
      "External id": 2488621,"Record function id": 0, "Ev Idx": 5420
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657900962.073, "dur": 2.974,
    "args": {
      "External id": 2488622,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5421
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657900962.817, "dur": 1.810,
    "args": {
      "External id": 2488623,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5422
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657900963.530, "dur": 0.975,
    "args": {
      "External id": 2488624,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5423
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657900968.942, "dur": 36749.658,
    "args": {
      "External id": 2488625,"Record function id": 0, "Sequence number": 24740425, "Fwd thread id": 1, "Ev Idx": 5424
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657900970.275, "dur": 36739.059,
    "args": {
      "External id": 2488626,"Sequence number": 24740425, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 5425
    }
  },
  {
    "ph": "f", "id": 223, "pid": 1336754, "tid": 1381179, "ts": 1514657900970.275,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.2)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657901038.921, "dur": 37.862,
    "args": {
      "External id": 2488627,"Record function id": 0, "Ev Idx": 5426
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.2)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657901084.793, "dur": 71.327,
    "args": {
      "External id": 2488628,"Record function id": 0, "Ev Idx": 5427
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.2)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657901161.931, "dur": 36538.967,
    "args": {
      "External id": 2488629,"Record function id": 0, "Ev Idx": 5428
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657901253.481, "dur": 6.926,
    "args": {
      "External id": 2488630,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5429
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657901270.809, "dur": 5.255,
    "args": {
      "External id": 2488631,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 5430
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514657901293.406, "dur": 35528.206,
    "args": {
      "External id": 2488632,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 5431
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514657901306.810, "dur": 35504.812,
    "args": {
      "External id": 2488633,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 5432
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657901360.908, "dur": 14.294,
    "args": {
      "External id": 2488634,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5433
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514657901381.342, "dur": 35389.736,
    "args": {
      "External id": 2488635,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 5434
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657901383.836, "dur": 35386.493,
    "args": {
      "External id": 2488636,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 5435
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657901388.052, "dur": 5.635,
    "args": {
      "External id": 2488637,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5436
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657901395.387, "dur": 35371.638,
    "args": {
      "External id": 2488638,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 5437
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657936910.798, "dur": 12.132,
    "args": {
      "External id": 2488639,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 5438
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657936916.521, "dur": 5.938,
    "args": {
      "External id": 2488640,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5439
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657936951.632, "dur": 431.010,
    "args": {
      "External id": 2488641,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 5440
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657936981.154, "dur": 395.918,
    "args": {
      "External id": 2488642,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 5441, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514657937030.219, "dur": 341.410,
    "args": {
      "External id": 2488643,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 5442
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657937402.814, "dur": 2.263,
    "args": {
      "External id": 2488644,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 5443, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657937469.468, "dur": 6.717,
    "args": {
      "External id": 2488645,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5444
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657937521.676, "dur": 3.900,
    "args": {
      "External id": 2488646,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5445
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657937562.421, "dur": 1.489,
    "args": {
      "External id": 2488647,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5446
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657937577.576, "dur": 1.245,
    "args": {
      "External id": 2488648,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5447
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657937590.449, "dur": 0.903,
    "args": {
      "External id": 2488649,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5448
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657937602.330, "dur": 2.747,
    "args": {
      "External id": 2488650,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5449
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657937615.676, "dur": 1.232,
    "args": {
      "External id": 2488651,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5450
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657937627.799, "dur": 1.437,
    "args": {
      "External id": 2488652,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5451
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657937639.045, "dur": 1.437,
    "args": {
      "External id": 2488653,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5452
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657937733.737, "dur": 2829.063,
    "args": {
      "External id": 2488654,"Record function id": 0, "Ev Idx": 5453
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.1)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657937754.122, "dur": 1051.057,
    "args": {
      "External id": 2488655,"Record function id": 0, "Ev Idx": 5454
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.1)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657937770.224, "dur": 360.316,
    "args": {
      "External id": 2488656,"Record function id": 0, "Ev Idx": 5455
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657937857.606, "dur": 3.995,
    "args": {
      "External id": 2488657,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 5456
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657937864.799, "dur": 3.024,
    "args": {
      "External id": 2488658,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5457
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657937869.918, "dur": 0.862,
    "args": {
      "External id": 2488659,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5458
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657937872.431, "dur": 0.953,
    "args": {
      "External id": 2488660,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5459
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657937875.091, "dur": 0.886,
    "args": {
      "External id": 2488661,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5460
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657937877.425, "dur": 1.132,
    "args": {
      "External id": 2488662,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5461
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657937880.413, "dur": 1.060,
    "args": {
      "External id": 2488663,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5462
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657937883.830, "dur": 0.691,
    "args": {
      "External id": 2488664,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5463
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657937885.897, "dur": 1.124,
    "args": {
      "External id": 2488665,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5464
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657937888.803, "dur": 3.194,
    "args": {
      "External id": 2488666,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5465
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657937909.690, "dur": 189.017,
    "args": {
      "External id": 2488667,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 5466
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657937925.188, "dur": 167.946,
    "args": {
      "External id": 2488668,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 5467
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657937942.290, "dur": 12.519,
    "args": {
      "External id": 2488669,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5468
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514657937957.944, "dur": 105.582,
    "args": {
      "External id": 2488670,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 5469
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657937961.664, "dur": 101.510,
    "args": {
      "External id": 2488671,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 5470
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657937965.224, "dur": 6.129,
    "args": {
      "External id": 2488672,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5471
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657937972.997, "dur": 89.107,
    "args": {
      "External id": 2488673,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 5472
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.0", "pid": 1336754, "tid": 1381179,
    "ts": 1514657938237.303, "dur": 560.402,
    "args": {
      "External id": 2488674,"Record function id": 0, "Ev Idx": 5473
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.0)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657938258.387, "dur": 527.637,
    "args": {
      "External id": 2488675,"Record function id": 0, "Ev Idx": 5474
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657938330.180, "dur": 6.039,
    "args": {
      "External id": 2488676,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5475
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1381179,
    "ts": 1514657938352.355, "dur": 29.087,
    "args": {
      "External id": 2488677,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 5476
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657938357.823, "dur": 1.600,
    "args": {
      "External id": 2488678,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5477
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657938361.587, "dur": 0.394,
    "args": {
      "External id": 2488679,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5478
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657938363.882, "dur": 0.493,
    "args": {
      "External id": 2488680,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5479
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657938365.714, "dur": 0.617,
    "args": {
      "External id": 2488681,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5480
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657938368.007, "dur": 0.629,
    "args": {
      "External id": 2488682,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5481
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657938370.267, "dur": 2.395,
    "args": {
      "External id": 2488683,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5482
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657938373.973, "dur": 0.711,
    "args": {
      "External id": 2488684,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5483
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657938376.289, "dur": 0.395,
    "args": {
      "External id": 2488685,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5484
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657938378.005, "dur": 0.444,
    "args": {
      "External id": 2488686,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5485
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657938392.612, "dur": 32.709,
    "args": {
      "External id": 2488687,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 5486
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1381179,
    "ts": 1514657938454.768, "dur": 114.789,
    "args": {
      "External id": 2488688,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "1", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 5487
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657938464.703, "dur": 2.587,
    "args": {
      "External id": 2488689,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5488
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1381179,
    "ts": 1514657938472.267, "dur": 9.762,
    "args": {
      "External id": 2488690,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 5489
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1381179,
    "ts": 1514657938476.422, "dur": 5.199,
    "args": {
      "External id": 2488691,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "12846080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 5490
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657938479.884, "dur": 0.656,
    "args": {
      "External id": 2488692,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 5491
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1381179,
    "ts": 1514657938488.515, "dur": 25.735,
    "args": {
      "External id": 2488693,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 5492
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657938490.224, "dur": 0.378,
    "args": {
      "External id": 2488694,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5493
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657938492.081, "dur": 0.874,
    "args": {
      "External id": 2488695,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6423296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5494
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657938494.277, "dur": 2.410,
    "args": {
      "External id": 2488696,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6947584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5495
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657938498.653, "dur": 0.358,
    "args": {
      "External id": 2488697,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7471872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5496
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657938500.845, "dur": 0.451,
    "args": {
      "External id": 2488698,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7996160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5497
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657938502.342, "dur": 0.347,
    "args": {
      "External id": 2488699,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8520448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5498
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657938504.113, "dur": 0.360,
    "args": {
      "External id": 2488700,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "8520704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5499
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657938506.067, "dur": 0.377,
    "args": {
      "External id": 2488701,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "9962496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5500
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657938507.514, "dur": 0.282,
    "args": {
      "External id": 2488702,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11404288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5501
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657938524.470, "dur": 36.412,
    "args": {
      "External id": 2488703,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 5502
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657938614.470, "dur": 107.922,
    "args": {
      "External id": 2488704,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 5503
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657938636.989, "dur": 82.384,
    "args": {
      "External id": 2488705,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 5504, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514657938647.163, "dur": 68.313,
    "args": {
      "External id": 2488706,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 5505
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657938737.452, "dur": 1.756,
    "args": {
      "External id": 2488707,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 5506, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657938813.047, "dur": 1708.262,
    "args": {
      "External id": 2488708,"Sequence number": 24740424, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 5507
    }
  },
  {
    "ph": "f", "id": 224, "pid": 1336754, "tid": 1381179, "ts": 1514657938813.047,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657938920.641, "dur": 147.198,
    "args": {
      "External id": 2488709,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 5508
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514657939113.170, "dur": 41.378,
    "args": {
      "External id": 2488710,"kernel_hash": "cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/fu/cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 5509
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657939172.229, "dur": 56.177,
    "args": {
      "External id": 2488711,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 5510
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657939238.031, "dur": 32.826,
    "args": {
      "External id": 2488712,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 5511
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657939277.730, "dur": 46.156,
    "args": {
      "External id": 2488713,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 5512
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657939333.070, "dur": 27.235,
    "args": {
      "External id": 2488714,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 5513
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657939368.805, "dur": 42.019,
    "args": {
      "External id": 2488715,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 5514
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657939435.121, "dur": 26.209,
    "args": {
      "External id": 2488716,"kernel_hash": "cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/wm/cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 5515
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514657939480.776, "dur": 31.563,
    "args": {
      "External id": 2488717,"kernel_hash": "cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/br/cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5516
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657939551.027, "dur": 26.901,
    "args": {
      "External id": 2488718,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 5517
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514657939593.713, "dur": 17.356,
    "args": {
      "External id": 2488719,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 5518
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657939621.711, "dur": 37.169,
    "args": {
      "External id": 2488720,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5519
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657939662.066, "dur": 34.856,
    "args": {
      "External id": 2488721,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5520
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657939725.217, "dur": 165.958,
    "args": {
      "External id": 2488722,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 5521
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657939799.785, "dur": 6.142,
    "args": {
      "External id": 2488723,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5522
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657939808.045, "dur": 2.561,
    "args": {
      "External id": 2488724,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5523
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657939925.682, "dur": 26.831,
    "args": {
      "External id": 2488725,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5524
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657939964.087, "dur": 16.137,
    "args": {
      "External id": 2488726,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5525
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940028.704, "dur": 46.341,
    "args": {
      "External id": 2488727,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5526
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940082.841, "dur": 41.625,
    "args": {
      "External id": 2488728,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5527
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940131.863, "dur": 35.040,
    "args": {
      "External id": 2488729,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5528
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940178.460, "dur": 38.058,
    "args": {
      "External id": 2488730,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5529
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940226.149, "dur": 24.264,
    "args": {
      "External id": 2488731,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5530
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940258.393, "dur": 31.395,
    "args": {
      "External id": 2488732,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5531
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940312.684, "dur": 24.388,
    "args": {
      "External id": 2488733,"kernel_hash": "c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/6w/c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 5532
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940364.919, "dur": 25.774,
    "args": {
      "External id": 2488734,"kernel_hash": "c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/4k/c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5533
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940407.538, "dur": 21.934,
    "args": {
      "External id": 2488735,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 5534
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940444.156, "dur": 15.721,
    "args": {
      "External id": 2488736,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 5535
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940476.251, "dur": 18.170,
    "args": {
      "External id": 2488737,"kernel_hash": "cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/eb/cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 5536
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940589.644, "dur": 16.016,
    "args": {
      "External id": 2488738,"Record function id": 0, "Ev Idx": 5537
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940593.219, "dur": 11.427,
    "args": {
      "External id": 2488739,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5538
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940597.685, "dur": 6.168,
    "args": {
      "External id": 2488740,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5539
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940599.901, "dur": 3.799,
    "args": {
      "External id": 2488741,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5540
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940609.585, "dur": 5.052,
    "args": {
      "External id": 2488742,"Record function id": 0, "Ev Idx": 5541
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940610.948, "dur": 3.283,
    "args": {
      "External id": 2488743,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5542
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940611.963, "dur": 1.689,
    "args": {
      "External id": 2488744,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5543
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940612.549, "dur": 1.034,
    "args": {
      "External id": 2488745,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5544
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940617.867, "dur": 4.726,
    "args": {
      "External id": 2488746,"Record function id": 0, "Ev Idx": 5545
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940619.294, "dur": 2.917,
    "args": {
      "External id": 2488747,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5546
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940620.080, "dur": 1.701,
    "args": {
      "External id": 2488748,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5547
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940620.637, "dur": 1.071,
    "args": {
      "External id": 2488749,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5548
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940625.952, "dur": 6.964,
    "args": {
      "External id": 2488750,"Record function id": 0, "Ev Idx": 5549
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940627.963, "dur": 4.508,
    "args": {
      "External id": 2488751,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5550
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940628.756, "dur": 3.321,
    "args": {
      "External id": 2488752,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5551
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940629.079, "dur": 2.932,
    "args": {
      "External id": 2488753,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5552
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940635.957, "dur": 4.589,
    "args": {
      "External id": 2488754,"Record function id": 0, "Ev Idx": 5553
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940637.387, "dur": 2.759,
    "args": {
      "External id": 2488755,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5554
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940638.269, "dur": 1.488,
    "args": {
      "External id": 2488756,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5555
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940638.728, "dur": 0.967,
    "args": {
      "External id": 2488757,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5556
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940643.650, "dur": 8.186,
    "args": {
      "External id": 2488758,"Record function id": 0, "Ev Idx": 5557
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940648.548, "dur": 2.873,
    "args": {
      "External id": 2488759,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5558
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940649.457, "dur": 1.528,
    "args": {
      "External id": 2488760,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5559
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940650.163, "dur": 0.712,
    "args": {
      "External id": 2488761,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5560
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940655.073, "dur": 4.234,
    "args": {
      "External id": 2488762,"Record function id": 0, "Ev Idx": 5561
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940656.420, "dur": 2.462,
    "args": {
      "External id": 2488763,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5562
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940657.200, "dur": 1.237,
    "args": {
      "External id": 2488764,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5563
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940657.570, "dur": 0.751,
    "args": {
      "External id": 2488765,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5564
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940662.301, "dur": 4.357,
    "args": {
      "External id": 2488766,"Record function id": 0, "Ev Idx": 5565
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940663.728, "dur": 2.549,
    "args": {
      "External id": 2488767,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5566
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940664.600, "dur": 1.137,
    "args": {
      "External id": 2488768,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5567
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940664.981, "dur": 0.660,
    "args": {
      "External id": 2488769,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5568
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940669.659, "dur": 4.328,
    "args": {
      "External id": 2488770,"Record function id": 0, "Ev Idx": 5569
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940671.075, "dur": 2.512,
    "args": {
      "External id": 2488771,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5570
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940671.575, "dur": 1.591,
    "args": {
      "External id": 2488772,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5571
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940672.182, "dur": 0.861,
    "args": {
      "External id": 2488773,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5572
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940677.832, "dur": 36617.966,
    "args": {
      "External id": 2488774,"Record function id": 0, "Sequence number": 24740423, "Fwd thread id": 1, "Ev Idx": 5573
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940679.129, "dur": 36607.234,
    "args": {
      "External id": 2488775,"Sequence number": 24740423, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 5574
    }
  },
  {
    "ph": "f", "id": 225, "pid": 1336754, "tid": 1381179, "ts": 1514657940679.129,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.1)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940708.153, "dur": 37.776,
    "args": {
      "External id": 2488776,"Record function id": 0, "Ev Idx": 5575
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.1)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940754.076, "dur": 71.054,
    "args": {
      "External id": 2488777,"Record function id": 0, "Ev Idx": 5576
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.1)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940830.697, "dur": 36447.786,
    "args": {
      "External id": 2488778,"Record function id": 0, "Ev Idx": 5577
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940922.884, "dur": 6.524,
    "args": {
      "External id": 2488779,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5578
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940938.856, "dur": 4.560,
    "args": {
      "External id": 2488780,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 5579
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940957.464, "dur": 35502.083,
    "args": {
      "External id": 2488781,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 5580
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514657940970.518, "dur": 35480.558,
    "args": {
      "External id": 2488782,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 5581
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657941056.914, "dur": 15.827,
    "args": {
      "External id": 2488783,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5582
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514657941079.247, "dur": 35332.528,
    "args": {
      "External id": 2488784,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 5583
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657941082.342, "dur": 35328.733,
    "args": {
      "External id": 2488785,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 5584
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657941086.277, "dur": 7.840,
    "args": {
      "External id": 2488786,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5585
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657941096.898, "dur": 35310.922,
    "args": {
      "External id": 2488787,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 5586
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657976561.294, "dur": 9.498,
    "args": {
      "External id": 2488788,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 5587
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657976564.142, "dur": 6.074,
    "args": {
      "External id": 2488789,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5588
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657976600.366, "dur": 348.541,
    "args": {
      "External id": 2488790,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 5589
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657976628.276, "dur": 315.950,
    "args": {
      "External id": 2488791,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 5590, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514657976639.854, "dur": 298.898,
    "args": {
      "External id": 2488792,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 5591
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514657976967.656, "dur": 2.106,
    "args": {
      "External id": 2488793,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 5592, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657977062.741, "dur": 7.213,
    "args": {
      "External id": 2488794,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5593
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657977114.850, "dur": 1.602,
    "args": {
      "External id": 2488795,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5594
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657977132.174, "dur": 1.125,
    "args": {
      "External id": 2488796,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5595
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657977145.991, "dur": 3.731,
    "args": {
      "External id": 2488797,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5596
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657977163.304, "dur": 1.028,
    "args": {
      "External id": 2488798,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5597
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657977176.882, "dur": 1.825,
    "args": {
      "External id": 2488799,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5598
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657977191.050, "dur": 1.827,
    "args": {
      "External id": 2488800,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5599
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657977204.230, "dur": 3.444,
    "args": {
      "External id": 2488801,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5600
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657977217.688, "dur": 1.068,
    "args": {
      "External id": 2488802,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5601
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657977310.981, "dur": 2212.946,
    "args": {
      "External id": 2488803,"Record function id": 0, "Ev Idx": 5602
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.0)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657977330.274, "dur": 479.235,
    "args": {
      "External id": 2488804,"Record function id": 0, "Ev Idx": 5603
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.0)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657977345.704, "dur": 349.213,
    "args": {
      "External id": 2488805,"Record function id": 0, "Ev Idx": 5604
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657977439.084, "dur": 4.404,
    "args": {
      "External id": 2488806,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 5605
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657977446.822, "dur": 1.114,
    "args": {
      "External id": 2488807,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5606
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657977449.491, "dur": 1.282,
    "args": {
      "External id": 2488808,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5607
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657977452.855, "dur": 0.834,
    "args": {
      "External id": 2488809,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5608
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657977455.409, "dur": 0.806,
    "args": {
      "External id": 2488810,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5609
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657977457.841, "dur": 1.077,
    "args": {
      "External id": 2488811,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5610
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657977460.308, "dur": 2.889,
    "args": {
      "External id": 2488812,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5611
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657977466.032, "dur": 1.098,
    "args": {
      "External id": 2488813,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5612
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657977468.505, "dur": 1.458,
    "args": {
      "External id": 2488814,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5613
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657977471.292, "dur": 1.143,
    "args": {
      "External id": 2488815,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5614
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657977489.345, "dur": 173.498,
    "args": {
      "External id": 2488816,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 5615
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657977507.236, "dur": 150.103,
    "args": {
      "External id": 2488817,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 5616
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657977522.604, "dur": 29.068,
    "args": {
      "External id": 2488818,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5617
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514657977555.957, "dur": 70.639,
    "args": {
      "External id": 2488819,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 5618
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657977558.504, "dur": 67.789,
    "args": {
      "External id": 2488820,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 5619
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657977562.801, "dur": 7.295,
    "args": {
      "External id": 2488821,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5620
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657977571.903, "dur": 53.901,
    "args": {
      "External id": 2488822,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 5621
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657977817.184, "dur": 1682.489,
    "args": {
      "External id": 2488823,"Sequence number": 24740422, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 5622
    }
  },
  {
    "ph": "f", "id": 226, "pid": 1336754, "tid": 1381179, "ts": 1514657977817.184,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657977922.364, "dur": 145.241,
    "args": {
      "External id": 2488824,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 5623
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514657978116.448, "dur": 47.696,
    "args": {
      "External id": 2488825,"kernel_hash": "cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/fu/cfuhkv7kka2c7zpla7ojjnsceqymafevjnpfblu4fjcq7zkig5bj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 5624
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657978182.702, "dur": 56.336,
    "args": {
      "External id": 2488826,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 5625
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657978248.945, "dur": 34.906,
    "args": {
      "External id": 2488827,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 5626
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657978291.088, "dur": 47.812,
    "args": {
      "External id": 2488828,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 5627
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657978347.963, "dur": 28.713,
    "args": {
      "External id": 2488829,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 5628
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657978384.622, "dur": 43.737,
    "args": {
      "External id": 2488830,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 5629
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657978451.488, "dur": 23.994,
    "args": {
      "External id": 2488831,"kernel_hash": "cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/wm/cwmdtyuwgj7j3fxg57ckso72wqi2qzdp23gsirbcv7772zguon5s.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 5630
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514657978493.389, "dur": 33.565,
    "args": {
      "External id": 2488832,"kernel_hash": "cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/br/cbr4fu2k5yr5uawq4iznngbgj5pxe5dndn6dpk7pftmhdoiq6bbz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5631
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657978587.967, "dur": 22.898,
    "args": {
      "External id": 2488833,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 5632
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514657978625.738, "dur": 16.801,
    "args": {
      "External id": 2488834,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 5633
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657978653.674, "dur": 35.895,
    "args": {
      "External id": 2488835,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5634
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657978693.150, "dur": 35.210,
    "args": {
      "External id": 2488836,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5635
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657978757.295, "dur": 172.350,
    "args": {
      "External id": 2488837,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 5636
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657978833.680, "dur": 6.376,
    "args": {
      "External id": 2488838,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5637
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657978842.284, "dur": 2.720,
    "args": {
      "External id": 2488839,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5638
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657978963.143, "dur": 63.076,
    "args": {
      "External id": 2488840,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5639
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979040.438, "dur": 18.262,
    "args": {
      "External id": 2488841,"kernel_hash": "cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/x2/cx2glzcaftzl4kfweo3o3hwtcmldzbfyqvcfcw2pz2jtvbtjqryt.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5640
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979069.365, "dur": 42.568,
    "args": {
      "External id": 2488842,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5641
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979121.518, "dur": 40.249,
    "args": {
      "External id": 2488843,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5642
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979168.898, "dur": 23.018,
    "args": {
      "External id": 2488844,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5643
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979198.277, "dur": 34.165,
    "args": {
      "External id": 2488845,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5644
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979239.040, "dur": 21.334,
    "args": {
      "External id": 2488846,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5645
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979270.317, "dur": 32.113,
    "args": {
      "External id": 2488847,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5646
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979318.853, "dur": 21.240,
    "args": {
      "External id": 2488848,"kernel_hash": "c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/6w/c6w57dwqlavni5blikci5m4csat5hnlhey5taj7vfkseu26zork7.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 5647
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979356.989, "dur": 25.086,
    "args": {
      "External id": 2488849,"kernel_hash": "c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/4k/c4kzku375wvjr3l7odspnz7rfogvuxnctfgzi5ygczeoztmm5agk.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5648
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979394.902, "dur": 18.514,
    "args": {
      "External id": 2488850,"kernel_hash": "ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/eo/ceozqyu7bzyhxbdlyjtvnat56d7ry6756gzuyvjf77ovbjuz4uhl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 5649
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979427.040, "dur": 16.708,
    "args": {
      "External id": 2488851,"kernel_hash": "ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn5gbptcig4tn232fiwehcb54hd6zxt5b2su6bhwqzwbeu53yqk.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 5650
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979454.807, "dur": 18.000,
    "args": {
      "External id": 2488852,"kernel_hash": "cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/eb/cebp6425kwg6qjs7vc5dpxqx2tgnwzzcolkzg6q26v5rzluozz5b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 5651
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979560.586, "dur": 16.967,
    "args": {
      "External id": 2488853,"Record function id": 0, "Ev Idx": 5652
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979564.726, "dur": 11.560,
    "args": {
      "External id": 2488854,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5653
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979569.015, "dur": 5.925,
    "args": {
      "External id": 2488855,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5654
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979570.675, "dur": 4.036,
    "args": {
      "External id": 2488856,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5655
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979581.508, "dur": 4.891,
    "args": {
      "External id": 2488857,"Record function id": 0, "Ev Idx": 5656
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979583.166, "dur": 2.805,
    "args": {
      "External id": 2488858,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5657
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979583.716, "dur": 1.755,
    "args": {
      "External id": 2488859,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5658
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979584.483, "dur": 0.920,
    "args": {
      "External id": 2488860,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5659
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979589.534, "dur": 5.315,
    "args": {
      "External id": 2488861,"Record function id": 0, "Ev Idx": 5660
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979591.257, "dur": 3.200,
    "args": {
      "External id": 2488862,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5661
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979592.043, "dur": 1.956,
    "args": {
      "External id": 2488863,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5662
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979592.541, "dur": 1.356,
    "args": {
      "External id": 2488864,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5663
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979598.077, "dur": 4.305,
    "args": {
      "External id": 2488865,"Record function id": 0, "Ev Idx": 5664
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979599.822, "dur": 2.157,
    "args": {
      "External id": 2488866,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5665
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979600.366, "dur": 1.182,
    "args": {
      "External id": 2488867,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5666
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979600.842, "dur": 0.641,
    "args": {
      "External id": 2488868,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5667
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979605.405, "dur": 6.051,
    "args": {
      "External id": 2488869,"Record function id": 0, "Ev Idx": 5668
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979606.883, "dur": 4.173,
    "args": {
      "External id": 2488870,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5669
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979607.406, "dur": 3.215,
    "args": {
      "External id": 2488871,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5670
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979607.710, "dur": 2.846,
    "args": {
      "External id": 2488872,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5671
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979614.628, "dur": 4.774,
    "args": {
      "External id": 2488873,"Record function id": 0, "Ev Idx": 5672
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979616.144, "dur": 2.843,
    "args": {
      "External id": 2488874,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5673
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979616.712, "dur": 1.689,
    "args": {
      "External id": 2488875,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5674
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979617.268, "dur": 0.839,
    "args": {
      "External id": 2488876,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5675
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979626.656, "dur": 4.728,
    "args": {
      "External id": 2488877,"Record function id": 0, "Ev Idx": 5676
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979628.395, "dur": 2.593,
    "args": {
      "External id": 2488878,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5677
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979629.195, "dur": 1.328,
    "args": {
      "External id": 2488879,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5678
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979629.690, "dur": 0.744,
    "args": {
      "External id": 2488880,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5679
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979634.596, "dur": 4.318,
    "args": {
      "External id": 2488881,"Record function id": 0, "Ev Idx": 5680
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979636.146, "dur": 2.336,
    "args": {
      "External id": 2488882,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5681
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979636.661, "dur": 1.387,
    "args": {
      "External id": 2488883,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5682
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979637.203, "dur": 0.749,
    "args": {
      "External id": 2488884,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5683
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979641.897, "dur": 4.139,
    "args": {
      "External id": 2488885,"Record function id": 0, "Ev Idx": 5684
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979643.095, "dur": 2.540,
    "args": {
      "External id": 2488886,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5685
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979643.941, "dur": 1.260,
    "args": {
      "External id": 2488887,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5686
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979644.461, "dur": 0.625,
    "args": {
      "External id": 2488888,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5687
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979650.014, "dur": 36555.844,
    "args": {
      "External id": 2488889,"Record function id": 0, "Sequence number": 24740421, "Fwd thread id": 1, "Ev Idx": 5688
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979651.254, "dur": 36546.367,
    "args": {
      "External id": 2488890,"Sequence number": 24740421, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 5689
    }
  },
  {
    "ph": "f", "id": 227, "pid": 1336754, "tid": 1381179, "ts": 1514657979651.254,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.0)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979680.797, "dur": 52.681,
    "args": {
      "External id": 2488891,"Record function id": 0, "Ev Idx": 5690
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.0)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979745.605, "dur": 75.243,
    "args": {
      "External id": 2488892,"Record function id": 0, "Ev Idx": 5691
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.0)", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979826.021, "dur": 36363.388,
    "args": {
      "External id": 2488893,"Record function id": 0, "Ev Idx": 5692
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979919.645, "dur": 6.553,
    "args": {
      "External id": 2488894,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5693
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979934.959, "dur": 4.584,
    "args": {
      "External id": 2488895,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 5694
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979956.725, "dur": 35403.251,
    "args": {
      "External id": 2488896,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 5695
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514657979969.544, "dur": 35381.535,
    "args": {
      "External id": 2488897,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 5696
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514657980079.525, "dur": 15.223,
    "args": {
      "External id": 2488898,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5697
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514657980101.414, "dur": 35206.322,
    "args": {
      "External id": 2488899,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 5698
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514657980103.728, "dur": 35203.334,
    "args": {
      "External id": 2488900,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 5699
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514657980109.067, "dur": 7.111,
    "args": {
      "External id": 2488901,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5700
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514657980118.889, "dur": 35185.337,
    "args": {
      "External id": 2488902,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 5701
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514658015448.088, "dur": 8.486,
    "args": {
      "External id": 2488903,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 5702
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514658015450.717, "dur": 5.522,
    "args": {
      "External id": 2488904,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5703
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514658015485.634, "dur": 359.132,
    "args": {
      "External id": 2488905,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 5704
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514658015512.202, "dur": 327.424,
    "args": {
      "External id": 2488906,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 5705, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514658015523.261, "dur": 310.895,
    "args": {
      "External id": 2488907,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 5706
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514658015864.458, "dur": 2.379,
    "args": {
      "External id": 2488908,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 5707, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514658015925.044, "dur": 6.794,
    "args": {
      "External id": 2488909,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5708
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514658015976.812, "dur": 4.002,
    "args": {
      "External id": 2488910,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5709
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514658016046.657, "dur": 1.961,
    "args": {
      "External id": 2488911,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5710
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514658016061.441, "dur": 0.927,
    "args": {
      "External id": 2488912,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5711
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514658016074.445, "dur": 0.926,
    "args": {
      "External id": 2488913,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5712
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514658016085.565, "dur": 3.015,
    "args": {
      "External id": 2488914,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5713
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514658016100.439, "dur": 1.131,
    "args": {
      "External id": 2488915,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5714
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514658016114.019, "dur": 0.832,
    "args": {
      "External id": 2488916,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5715
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514658016126.424, "dur": 1.077,
    "args": {
      "External id": 2488917,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5716
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514658016222.422, "dur": 285.421,
    "args": {
      "External id": 2488918,"Record function id": 0, "Sequence number": 24740420, "Fwd thread id": 1, "Ev Idx": 5717
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336754, "tid": 1381179,
    "ts": 1514658016225.325, "dur": 274.615,
    "args": {
      "External id": 2488919,"Sequence number": 24740420, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 5718
    }
  },
  {
    "ph": "f", "id": 228, "pid": 1336754, "tid": 1381179, "ts": 1514658016225.325,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_embedding_dense_backward_0", "pid": 1336754, "tid": 1381179,
    "ts": 1514658016347.121, "dur": 47.816,
    "args": {
      "External id": 2488920,"kernel_hash": "c6aziuyp6fagrmcedyk7wtb6g56x7av3etx5bs4re25wo6lnvnsr", "grid": "grid(65536000,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "65536000"], "kernel_file": "/tmp/torchinductor_cvm/6a/c6aziuyp6fagrmcedyk7wtb6g56x7av3etx5bs4re25wo6lnvnsr.py", "kernel_backend": "triton", "Input type": ["float", "Scalar"], "Input Strides": [[2048, 1], []], "Input Dims": [[32000, 2048], []], "Ev Idx": 5719
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_embedding_dense_backward_1", "pid": 1336754, "tid": 1381179,
    "ts": 1514658016412.664, "dur": 31.968,
    "args": {
      "External id": 2488921,"kernel_hash": "cltgraj4uipr7sa5n4ns4uyvanquunkc3ybdvrfy5pw2v2cf6swt", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/lt/cltgraj4uipr7sa5n4ns4uyvanquunkc3ybdvrfy5pw2v2cf6swt.py", "kernel_backend": "triton", "Input type": ["long int", "c10::BFloat16", "float", "Scalar"], "Input Strides": [[4096, 1], [8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096], [16, 4096, 2048], [32000, 2048], []], "Ev Idx": 5720
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_embedding_dense_backward_2", "pid": 1336754, "tid": 1381179,
    "ts": 1514658016461.448, "dur": 23.838,
    "args": {
      "External id": 2488922,"kernel_hash": "chiuzhe4yx4geeiep36ony2tqipsfvpo7rwythdxblopxivdgss3", "grid": "grid(65536000,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "65536000"], "kernel_file": "/tmp/torchinductor_cvm/hi/chiuzhe4yx4geeiep36ony2tqipsfvpo7rwythdxblopxivdgss3.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 5721
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514658016517.116, "dur": 30.777,
    "args": {
      "External id": 2488923,"Record function id": 0, "Ev Idx": 5722
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336754, "tid": 1381179,
    "ts": 1514658016520.092, "dur": 26.192,
    "args": {
      "External id": 2488924,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 5723
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514658016523.561, "dur": 21.263,
    "args": {
      "External id": 2488925,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 5724
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1381179,
    "ts": 1514658016525.178, "dur": 19.153,
    "args": {
      "External id": 2488926,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 5725
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::root_post_backward_callback", "pid": 1336754, "tid": 1381179,
    "ts": 1514658016572.264, "dur": 10499.976,
    "args": {
      "External id": 2488927,"Record function id": 0, "Ev Idx": 5726
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate", "pid": 1336754, "tid": 1381179,
    "ts": 1514658016593.407, "dur": 40.006,
    "args": {
      "External id": 2488928,"Record function id": 0, "Ev Idx": 5727
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard", "pid": 1336754, "tid": 1381179,
    "ts": 1514658016640.244, "dur": 319.365,
    "args": {
      "External id": 2488929,"Record function id": 0, "Ev Idx": 5728
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce", "pid": 1336754, "tid": 1381179,
    "ts": 1514658016966.159, "dur": 9783.176,
    "args": {
      "External id": 2488930,"Record function id": 0, "Ev Idx": 5729
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514658017127.442, "dur": 6.888,
    "args": {
      "External id": 2488931,"Record function id": 0, "Concrete Inputs": ["[336611328]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5730
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1381179,
    "ts": 1514658017145.263, "dur": 5.442,
    "args": {
      "External id": 2488932,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[336611328], []], "Ev Idx": 5731
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514658017172.383, "dur": 8200.783,
    "args": {
      "External id": 2488933,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[], [], [], [42076416, 1]], "Input Dims": [[], [], [], [8, 42076416]], "Ev Idx": 5732
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336754, "tid": 1381179,
    "ts": 1514658017190.758, "dur": 8168.708,
    "args": {
      "External id": 2488934,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[], [], [], [42076416, 1]], "Input Dims": [[], [], [], [8, 42076416]], "Ev Idx": 5733
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514658017545.501, "dur": 19.558,
    "args": {
      "External id": 2488935,"Record function id": 0, "Concrete Inputs": ["[82421]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5734
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1381179,
    "ts": 1514658017662.088, "dur": 7646.229,
    "args": {
      "External id": 2488936,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[82421], [], [], [], [], [], [], []], "Ev Idx": 5735
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1381179,
    "ts": 1514658017664.847, "dur": 7642.521,
    "args": {
      "External id": 2488937,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[82421], [], [], [], [], [], []], "Ev Idx": 5736
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514658017670.079, "dur": 12.349,
    "args": {
      "External id": 2488938,"Record function id": 0, "Concrete Inputs": ["[82421]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5737
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1381179,
    "ts": 1514658017684.748, "dur": 7617.709,
    "args": {
      "External id": 2488939,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[82421], [82421], []], "Ev Idx": 5738
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514658025515.542, "dur": 12.008,
    "args": {
      "External id": 2488940,"Record function id": 0, "Concrete Inputs": ["", "[42076416]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[336611328], [], [], [], [], []], "Ev Idx": 5739
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1381179,
    "ts": 1514658025519.623, "dur": 7.403,
    "args": {
      "External id": 2488941,"Record function id": 0, "Concrete Inputs": ["[42076416]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5740
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336754, "tid": 1381179,
    "ts": 1514658025573.267, "dur": 484.567,
    "args": {
      "External id": 2488942,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[42076416], [336611328], [], [], [], []], "Ev Idx": 5741
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514658025599.310, "dur": 452.665,
    "args": {
      "External id": 2488943,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 42076416, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[336611328], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 5742, "In msg nelems": 336611328
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336754, "tid": 1381179,
    "ts": 1514658025615.645, "dur": 429.948,
    "args": {
      "External id": 2488944,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[336611328]], "Ev Idx": 5743
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1381179,
    "ts": 1514658026078.655, "dur": 2.827,
    "args": {
      "External id": 2488945,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 5744, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514658026145.375, "dur": 7.319,
    "args": {
      "External id": 2488946,"Record function id": 0, "Concrete Inputs": ["", "[4000, 2048]", "[2048, 1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5745
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514658026196.058, "dur": 1.614,
    "args": {
      "External id": 2488947,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8192000"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5746
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514658026213.162, "dur": 1.174,
    "args": {
      "External id": 2488948,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "8192256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5747
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514658026226.849, "dur": 2.808,
    "args": {
      "External id": 2488949,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "8716544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5748
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514658026241.436, "dur": 1.509,
    "args": {
      "External id": 2488950,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "9240832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5749
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514658026253.676, "dur": 1.105,
    "args": {
      "External id": 2488951,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "9765120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5750
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514658026266.374, "dur": 1.213,
    "args": {
      "External id": 2488952,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "10289408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5751
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514658026280.386, "dur": 4.739,
    "args": {
      "External id": 2488953,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "10289664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5752
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514658026297.992, "dur": 1.040,
    "args": {
      "External id": 2488954,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "11731456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5753
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514658026308.597, "dur": 1.487,
    "args": {
      "External id": 2488955,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "13173248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5754
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514658026320.719, "dur": 1.352,
    "args": {
      "External id": 2488956,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "14615040"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5755
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514658026333.047, "dur": 3.031,
    "args": {
      "External id": 2488957,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "14615296"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5756
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514658026347.035, "dur": 1.244,
    "args": {
      "External id": 2488958,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "15139584"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5757
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514658026359.109, "dur": 1.861,
    "args": {
      "External id": 2488959,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "15663872"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5758
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514658026371.340, "dur": 1.378,
    "args": {
      "External id": 2488960,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "16188160"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5759
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514658026382.807, "dur": 3.327,
    "args": {
      "External id": 2488961,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "16712448"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5760
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514658026396.620, "dur": 1.265,
    "args": {
      "External id": 2488962,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "16712704"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5761
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514658026408.116, "dur": 1.568,
    "args": {
      "External id": 2488963,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "18154496"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5762
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514658026420.325, "dur": 1.774,
    "args": {
      "External id": 2488964,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "19596288"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5763
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514658026431.778, "dur": 2.901,
    "args": {
      "External id": 2488965,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21038080"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5764
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514658026444.296, "dur": 1.362,
    "args": {
      "External id": 2488966,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "21038336"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5765
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514658026456.059, "dur": 1.234,
    "args": {
      "External id": 2488967,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "21562624"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5766
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514658026467.177, "dur": 1.059,
    "args": {
      "External id": 2488968,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "22086912"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5767
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514658026479.337, "dur": 3.323,
    "args": {
      "External id": 2488969,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "22611200"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5768
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514658026492.724, "dur": 1.225,
    "args": {
      "External id": 2488970,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "23135488"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5769
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514658026504.508, "dur": 1.427,
    "args": {
      "External id": 2488971,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "23135744"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5770
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514658026516.021, "dur": 1.553,
    "args": {
      "External id": 2488972,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "24577536"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5771
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514658026540.690, "dur": 3.826,
    "args": {
      "External id": 2488973,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "26019328"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5772
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514658026559.316, "dur": 1.607,
    "args": {
      "External id": 2488974,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27461120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5773
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514658026571.151, "dur": 1.267,
    "args": {
      "External id": 2488975,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "27461376"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5774
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514658026583.458, "dur": 1.447,
    "args": {
      "External id": 2488976,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "27985664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5775
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514658026595.986, "dur": 3.565,
    "args": {
      "External id": 2488977,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "28509952"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5776
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514658026609.550, "dur": 1.190,
    "args": {
      "External id": 2488978,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "29034240"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5777
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514658026621.850, "dur": 1.561,
    "args": {
      "External id": 2488979,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "29558528"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5778
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514658026634.410, "dur": 1.395,
    "args": {
      "External id": 2488980,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "29558784"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5779
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514658026646.762, "dur": 3.360,
    "args": {
      "External id": 2488981,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "31000576"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5780
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514658026661.555, "dur": 1.617,
    "args": {
      "External id": 2488982,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "32442368"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5781
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514658026673.565, "dur": 1.134,
    "args": {
      "External id": 2488983,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "33884160"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5782
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1381179,
    "ts": 1514658026685.391, "dur": 1.142,
    "args": {
      "External id": 2488984,"Record function id": 0, "Concrete Inputs": ["", "[4000, 2048]", "[2048, 1]", "33884416"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5783
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "ProfilerStep#102911", "pid": 1336754, "tid": 1336754,
    "ts": 1514655936944.843, "dur": 2107955.411,
    "args": {
      "External id": 2478081,"Record function id": 0, "Ev Idx": 5784
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "Optimizer.zero_grad#AdamW.zero_grad", "pid": 1336754, "tid": 1336754,
    "ts": 1514655936976.166, "dur": 718.432,
    "args": {
      "External id": 2478082,"Record function id": 0, "Ev Idx": 5785
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "enumerate(DataLoader)#_StatefulMultiProcessingDataLoaderIter.__next__", "pid": 1336754, "tid": 1336754,
    "ts": 1514655937737.192, "dur": 1902.508,
    "args": {
      "External id": 2478083,"Record function id": 0, "Ev Idx": 5786
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514655938709.158, "dur": 7.435,
    "args": {
      "External id": 2478084,"Record function id": 0, "Concrete Inputs": ["[0]", "4", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5787
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::set_", "pid": 1336754, "tid": 1336754,
    "ts": 1514655938738.005, "dur": 6.356,
    "args": {
      "External id": 2478085,"Sequence number": 24740420, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "0", "[16, 8192]", "[8192, 1]"], "Input type": ["long int", "", "Scalar", "ScalarList", "ScalarList"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[0], [], [], [], []], "Ev Idx": 5788
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514655939147.981, "dur": 2.189,
    "args": {
      "External id": 2478086,"Record function id": 0, "Concrete Inputs": ["[0]", "4", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5789
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::set_", "pid": 1336754, "tid": 1336754,
    "ts": 1514655939157.054, "dur": 2.499,
    "args": {
      "External id": 2478087,"Sequence number": 24740420, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "0", "[16, 8192]", "[8192, 1]"], "Input type": ["long int", "", "Scalar", "ScalarList", "ScalarList"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[0], [], [], [], []], "Ev Idx": 5790
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514655939522.036, "dur": 1.507,
    "args": {
      "External id": 2478088,"Record function id": 0, "Concrete Inputs": ["[0]", "4", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5791
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::set_", "pid": 1336754, "tid": 1336754,
    "ts": 1514655939544.148, "dur": 2.368,
    "args": {
      "External id": 2478089,"Sequence number": 24740420, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "0", "[16, 8192]", "[8192, 1]"], "Input type": ["long int", "", "Scalar", "ScalarList", "ScalarList"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[0], [], [], [], []], "Ev Idx": 5792
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514655940210.970, "dur": 13.838,
    "args": {
      "External id": 2478090,"Sequence number": 24740420, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1], [], [], [], []], "Input Dims": [[16, 8192], [], [], [], []], "Ev Idx": 5793
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655940218.482, "dur": 2.417,
    "args": {
      "External id": 2478091,"Record function id": 0, "Concrete Inputs": ["", "[16, 8192]", "[8192, 1]", "0"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1], [], [], []], "Input Dims": [[16, 8192], [], [], []], "Ev Idx": 5794
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514655940226.204, "dur": 4.048,
    "args": {
      "External id": 2478092,"Sequence number": 24740420, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "0", "4096", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1], [], [], [], []], "Input Dims": [[16, 8192], [], [], [], []], "Ev Idx": 5795
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655940228.163, "dur": 1.089,
    "args": {
      "External id": 2478093,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096]", "[8192, 1]", "0"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1], [], [], []], "Input Dims": [[16, 8192], [], [], []], "Ev Idx": 5796
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514655940256.980, "dur": 576.460,
    "args": {
      "External id": 2478094,"Sequence number": 24740420, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "4", "0", "", "", "False", "False", ""], "Input type": ["long int", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[8192, 1], [], [], [], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], [], [], [], []], "Ev Idx": 5797
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514655940264.058, "dur": 568.449,
    "args": {
      "External id": 2478095,"Sequence number": 24740420, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "4", "0", "", "", "False", ""], "Input type": ["long int", "Scalar", "Scalar", "", "", "Scalar", ""], "Input Strides": [[8192, 1], [], [], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], [], [], []], "Ev Idx": 5798
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655940271.926, "dur": 9.924,
    "args": {
      "External id": 2478096,"Record function id": 0, "Concrete Inputs": ["[16, 4096]", "[4096, 1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5799
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514655940284.346, "dur": 546.310,
    "args": {
      "External id": 2478097,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[4096, 1], [8192, 1], []], "Input Dims": [[16, 4096], [16, 4096], []], "Ev Idx": 5800
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514655940293.546, "dur": 0.632,
    "args": {
      "External id": 2478098,"Record function id": 0, "Concrete Inputs": ["", "4", "False", "False", ""], "Input type": ["long int", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[8192, 1], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], []], "Ev Idx": 5801
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::expand_as", "pid": 1336754, "tid": 1336754,
    "ts": 1514655940297.416, "dur": 7.732,
    "args": {
      "External id": 2478099,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["long int", "long int"], "Input Strides": [[8192, 1], [4096, 1]], "Input Dims": [[16, 4096], [16, 4096]], "Ev Idx": 5802
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::expand", "pid": 1336754, "tid": 1336754,
    "ts": 1514655940301.738, "dur": 3.259,
    "args": {
      "External id": 2478100,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096]", "False"], "Input type": ["long int", "ScalarList", "Scalar"], "Input Strides": [[8192, 1], [], []], "Input Dims": [[16, 4096], [], []], "Ev Idx": 5803
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655940304.046, "dur": 0.596,
    "args": {
      "External id": 2478101,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096]", "[8192, 1]", ""], "Input type": ["long int", "ScalarList", "ScalarList", ""], "Input Strides": [[8192, 1], [], [], []], "Input Dims": [[16, 4096], [], [], []], "Ev Idx": 5804
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::contiguous", "pid": 1336754, "tid": 1336754,
    "ts": 1514655940307.324, "dur": 273.503,
    "args": {
      "External id": 2478102,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["long int", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 5805
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336754, "tid": 1336754,
    "ts": 1514655940309.900, "dur": 270.579,
    "args": {
      "External id": 2478103,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["long int", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 5806
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514655940312.011, "dur": 13.954,
    "args": {
      "External id": 2478104,"Record function id": 0, "Concrete Inputs": ["", "4", "0", "", "", "0"], "Input type": ["long int", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[8192, 1], [], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], [], []], "Ev Idx": 5807
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514655940313.922, "dur": 11.508,
    "args": {
      "External id": 2478105,"Record function id": 0, "Concrete Inputs": ["[16, 4096]", "4", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5808
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514655940326.913, "dur": 252.139,
    "args": {
      "External id": 2478106,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[4096, 1], [8192, 1], []], "Input Dims": [[16, 4096], [16, 4096], []], "Ev Idx": 5809
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514655940583.942, "dur": 242.707,
    "args": {
      "External id": 2478107,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[4096, 1], [4096, 1], []], "Input Dims": [[16, 4096], [16, 4096], []], "Ev Idx": 5810
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514655940851.978, "dur": 774.861,
    "args": {
      "External id": 2478108,"Sequence number": 24740420, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "4", "0", "", "", "False", "False", ""], "Input type": ["long int", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[8192, 1], [], [], [], [], [], [], []], "Input Dims": [[16, 8192], [], [], [], [], [], [], []], "Ev Idx": 5811
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514655940853.906, "dur": 772.169,
    "args": {
      "External id": 2478109,"Sequence number": 24740420, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "4", "0", "", "", "False", ""], "Input type": ["long int", "Scalar", "Scalar", "", "", "Scalar", ""], "Input Strides": [[8192, 1], [], [], [], [], [], []], "Input Dims": [[16, 8192], [], [], [], [], [], []], "Ev Idx": 5812
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655940860.200, "dur": 9.363,
    "args": {
      "External id": 2478110,"Record function id": 0, "Concrete Inputs": ["[16, 8192]", "[8192, 1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5813
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514655940870.505, "dur": 751.475,
    "args": {
      "External id": 2478111,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[8192, 1], [8192, 1], []], "Input Dims": [[16, 8192], [16, 8192], []], "Ev Idx": 5814
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::arange", "pid": 1336754, "tid": 1336754,
    "ts": 1514655941657.337, "dur": 52.760,
    "args": {
      "External id": 2478112,"Record function id": 0, "Concrete Inputs": ["0", "4096", "", "", "", "False"], "Input type": ["Scalar", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5815
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514655941662.826, "dur": 5.552,
    "args": {
      "External id": 2478113,"Record function id": 0, "Concrete Inputs": ["[0]", "4", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5816
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::arange", "pid": 1336754, "tid": 1336754,
    "ts": 1514655941671.412, "dur": 38.310,
    "args": {
      "External id": 2478114,"Record function id": 0, "Concrete Inputs": ["0", "4096", "1", ""], "Input type": ["Scalar", "Scalar", "Scalar", "long int"], "Input Strides": [[], [], [], [1]], "Input Dims": [[], [], [], [0]], "Ev Idx": 5817
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514655941676.685, "dur": 7.381,
    "args": {
      "External id": 2478115,"Record function id": 0, "Concrete Inputs": ["", "[4096]", ""], "Input type": ["long int", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 5818
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::repeat", "pid": 1336754, "tid": 1336754,
    "ts": 1514655941722.751, "dur": 66.940,
    "args": {
      "External id": 2478116,"Sequence number": 24740420, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 1]"], "Input type": ["long int", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4096], []], "Ev Idx": 5819
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::expand", "pid": 1336754, "tid": 1336754,
    "ts": 1514655941727.435, "dur": 6.173,
    "args": {
      "External id": 2478117,"Record function id": 0, "Concrete Inputs": ["", "[1, 4096]", "False"], "Input type": ["long int", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[4096], [], []], "Ev Idx": 5820
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655941731.722, "dur": 1.656,
    "args": {
      "External id": 2478118,"Record function id": 0, "Concrete Inputs": ["", "[1, 4096]", "[4096, 1]", ""], "Input type": ["long int", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[4096], [], [], []], "Ev Idx": 5821
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514655941734.561, "dur": 4.652,
    "args": {
      "External id": 2478119,"Record function id": 0, "Concrete Inputs": ["[16, 4096]", "4", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5822
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336754, "tid": 1336754,
    "ts": 1514655941741.839, "dur": 3.622,
    "args": {
      "External id": 2478120,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["long int"], "Input Strides": [[4096, 1]], "Input Dims": [[16, 4096]], "Ev Idx": 5823
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unfold", "pid": 1336754, "tid": 1336754,
    "ts": 1514655941749.679, "dur": 4.873,
    "args": {
      "External id": 2478121,"Record function id": 0, "Concrete Inputs": ["", "0", "1", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar"], "Input Strides": [[4096, 1], [], [], []], "Input Dims": [[16, 4096], [], [], []], "Ev Idx": 5824
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655941753.665, "dur": 0.479,
    "args": {
      "External id": 2478122,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 1]", "[4096, 1, 4096]", ""], "Input type": ["long int", "ScalarList", "ScalarList", ""], "Input Strides": [[4096, 1], [], [], []], "Input Dims": [[16, 4096], [], [], []], "Ev Idx": 5825
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unfold", "pid": 1336754, "tid": 1336754,
    "ts": 1514655941755.256, "dur": 3.790,
    "args": {
      "External id": 2478123,"Record function id": 0, "Concrete Inputs": ["", "1", "4096", "4096"], "Input type": ["long int", "Scalar", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 4096], [], [], []], "Input Dims": [[16, 4096, 1], [], [], []], "Ev Idx": 5826
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655941757.895, "dur": 0.890,
    "args": {
      "External id": 2478124,"Record function id": 0, "Concrete Inputs": ["", "[16, 1, 1, 4096]", "[4096, 4096, 4096, 1]", ""], "Input type": ["long int", "ScalarList", "ScalarList", ""], "Input Strides": [[4096, 1, 4096], [], [], []], "Input Dims": [[16, 4096, 1], [], [], []], "Ev Idx": 5827
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::expand_as", "pid": 1336754, "tid": 1336754,
    "ts": 1514655941760.750, "dur": 3.424,
    "args": {
      "External id": 2478125,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["long int", "long int"], "Input Strides": [[4096, 1], [4096, 4096, 4096, 1]], "Input Dims": [[1, 4096], [16, 1, 1, 4096]], "Ev Idx": 5828
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::expand", "pid": 1336754, "tid": 1336754,
    "ts": 1514655941761.786, "dur": 2.287,
    "args": {
      "External id": 2478126,"Record function id": 0, "Concrete Inputs": ["", "[16, 1, 1, 4096]", "False"], "Input type": ["long int", "ScalarList", "Scalar"], "Input Strides": [[4096, 1], [], []], "Input Dims": [[1, 4096], [], []], "Ev Idx": 5829
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655941762.979, "dur": 1.001,
    "args": {
      "External id": 2478127,"Record function id": 0, "Concrete Inputs": ["", "[16, 1, 1, 4096]", "[0, 4096, 4096, 1]", ""], "Input type": ["long int", "ScalarList", "ScalarList", ""], "Input Strides": [[4096, 1], [], [], []], "Input Dims": [[1, 4096], [], [], []], "Ev Idx": 5830
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514655941765.015, "dur": 23.898,
    "args": {
      "External id": 2478128,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[4096, 4096, 4096, 1], [0, 4096, 4096, 1], []], "Input Dims": [[16, 1, 1, 4096], [16, 1, 1, 4096], []], "Ev Idx": 5831
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514655941796.905, "dur": 28.892,
    "args": {
      "External id": 2478129,"Sequence number": 24740420, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "3", "False", "False", ""], "Input type": ["long int", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[4096, 1], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], []], "Ev Idx": 5832
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514655941798.377, "dur": 27.246,
    "args": {
      "External id": 2478130,"Sequence number": 24740420, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "3", "", "", "", "False", ""], "Input type": ["long int", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[4096, 1], [], [], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], [], [], []], "Ev Idx": 5833
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655941805.522, "dur": 3.226,
    "args": {
      "External id": 2478131,"Record function id": 0, "Concrete Inputs": ["[16, 4096]", "[4096, 1]", "3", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5834
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514655941809.341, "dur": 15.915,
    "args": {
      "External id": 2478132,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["int", "long int", "Scalar"], "Input Strides": [[4096, 1], [4096, 1], []], "Input Dims": [[16, 4096], [16, 4096], []], "Ev Idx": 5835
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::root_pre_forward", "pid": 1336754, "tid": 1336754,
    "ts": 1514655941938.519, "dur": 190.582,
    "args": {
      "External id": 2478133,"Record function id": 0, "Ev Idx": 5836
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::inputs_to_device", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942062.397, "dur": 56.531,
    "args": {
      "External id": 2478134,"Record function id": 0, "Ev Idx": 5837
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942136.322, "dur": 41.455,
    "args": {
      "External id": 2478135,"Record function id": 0, "Ev Idx": 5838
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942186.750, "dur": 10061.889,
    "args": {
      "External id": 2478136,"Record function id": 0, "Ev Idx": 5839
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942195.974, "dur": 1235.810,
    "args": {
      "External id": 2478137,"Record function id": 0, "Ev Idx": 5840
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942311.255, "dur": 7.313,
    "args": {
      "External id": 2478138,"Record function id": 0, "Concrete Inputs": ["[42076416]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5841
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942338.042, "dur": 105.822,
    "args": {
      "External id": 2478139,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["c10::BFloat16", "", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[42076416], [], []], "Ev Idx": 5842
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942342.704, "dur": 1.787,
    "args": {
      "External id": 2478140,"Record function id": 0, "Concrete Inputs": ["", "[8192000]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5843
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942349.569, "dur": 0.304,
    "args": {
      "External id": 2478141,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8192000"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5844
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942350.609, "dur": 0.241,
    "args": {
      "External id": 2478142,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "8192256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5845
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942351.511, "dur": 1.784,
    "args": {
      "External id": 2478143,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "8716544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5846
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942357.393, "dur": 0.407,
    "args": {
      "External id": 2478144,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "9240832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5847
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942358.769, "dur": 0.417,
    "args": {
      "External id": 2478145,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "9765120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5848
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942359.743, "dur": 1.761,
    "args": {
      "External id": 2478146,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "10289408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5849
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942363.897, "dur": 0.192,
    "args": {
      "External id": 2478147,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "10289664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5850
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942364.857, "dur": 0.458,
    "args": {
      "External id": 2478148,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11731456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5851
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942369.004, "dur": 0.178,
    "args": {
      "External id": 2478149,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "13173248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5852
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942369.730, "dur": 0.366,
    "args": {
      "External id": 2478150,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "14615040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5853
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942370.666, "dur": 1.280,
    "args": {
      "External id": 2478151,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "14615296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5854
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942374.255, "dur": 0.351,
    "args": {
      "External id": 2478152,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "15139584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5855
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942375.482, "dur": 0.159,
    "args": {
      "External id": 2478153,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "15663872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5856
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942376.399, "dur": 2.291,
    "args": {
      "External id": 2478154,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "16188160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5857
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942380.494, "dur": 0.175,
    "args": {
      "External id": 2478155,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "16712448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5858
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942381.331, "dur": 0.175,
    "args": {
      "External id": 2478156,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "16712704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5859
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942385.287, "dur": 0.275,
    "args": {
      "External id": 2478157,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "18154496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5860
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942386.186, "dur": 0.151,
    "args": {
      "External id": 2478158,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "19596288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5861
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942386.906, "dur": 1.336,
    "args": {
      "External id": 2478159,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21038080"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5862
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942390.902, "dur": 0.476,
    "args": {
      "External id": 2478160,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "21038336"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5863
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942391.935, "dur": 0.359,
    "args": {
      "External id": 2478161,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "21562624"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5864
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942392.955, "dur": 2.121,
    "args": {
      "External id": 2478162,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "22086912"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5865
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942397.489, "dur": 0.279,
    "args": {
      "External id": 2478163,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "22611200"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5866
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942398.302, "dur": 0.480,
    "args": {
      "External id": 2478164,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "23135488"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5867
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942402.405, "dur": 0.275,
    "args": {
      "External id": 2478165,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "23135744"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5868
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942403.246, "dur": 0.442,
    "args": {
      "External id": 2478166,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24577536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5869
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942404.329, "dur": 0.910,
    "args": {
      "External id": 2478167,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "26019328"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5870
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942408.878, "dur": 0.263,
    "args": {
      "External id": 2478168,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27461120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5871
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942409.803, "dur": 0.665,
    "args": {
      "External id": 2478169,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27461376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5872
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942411.037, "dur": 2.011,
    "args": {
      "External id": 2478170,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27985664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5873
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942415.427, "dur": 0.350,
    "args": {
      "External id": 2478171,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "28509952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5874
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942416.399, "dur": 0.348,
    "args": {
      "External id": 2478172,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "29034240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5875
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942421.113, "dur": 0.141,
    "args": {
      "External id": 2478173,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "29558528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5876
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942421.732, "dur": 0.153,
    "args": {
      "External id": 2478174,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29558784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5877
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942422.415, "dur": 1.086,
    "args": {
      "External id": 2478175,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "31000576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5878
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942425.981, "dur": 0.226,
    "args": {
      "External id": 2478176,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "32442368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5879
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942427.055, "dur": 0.353,
    "args": {
      "External id": 2478177,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "33884160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5880
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942427.941, "dur": 1.775,
    "args": {
      "External id": 2478178,"Record function id": 0, "Concrete Inputs": ["", "[8192000]", "[1]", "33884416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5881
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942467.198, "dur": 91.545,
    "args": {
      "External id": 2478179,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[], [], []], "Input Dims": [[], [], []], "Ev Idx": 5882
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942640.996, "dur": 245.782,
    "args": {
      "External id": 2478180,"Record function id": 0, "Concrete Inputs": ["", "", "42076416", "8", "1", "15", ""], "Input type": ["TensorList", "", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], []], "Ev Idx": 5883
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942659.097, "dur": 5.161,
    "args": {
      "External id": 2478181,"Record function id": 0, "Concrete Inputs": ["[336611328]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5884
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942671.490, "dur": 13.060,
    "args": {
      "External id": 2478182,"Record function id": 0, "Concrete Inputs": ["", "0", "42076416", "42076416"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[336611328], [], [], []], "Ev Idx": 5885
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942675.891, "dur": 8.239,
    "args": {
      "External id": 2478183,"Record function id": 0, "Concrete Inputs": ["", "0", "42076416", "84152832", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[336611328], [], [], [], []], "Ev Idx": 5886
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942680.877, "dur": 0.853,
    "args": {
      "External id": 2478184,"Record function id": 0, "Concrete Inputs": ["", "[42076416]", "[1]", "42076416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[336611328], [], [], []], "Ev Idx": 5887
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942692.076, "dur": 98.969,
    "args": {
      "External id": 2478185,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["c10::BFloat16", "", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[42076416], [], []], "Ev Idx": 5888
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942694.189, "dur": 0.655,
    "args": {
      "External id": 2478186,"Record function id": 0, "Concrete Inputs": ["", "[8192000]", "[1]", "42076416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5889
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942695.815, "dur": 0.204,
    "args": {
      "External id": 2478187,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "50268416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5890
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942698.155, "dur": 1.452,
    "args": {
      "External id": 2478188,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "50268672"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5891
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942700.073, "dur": 0.683,
    "args": {
      "External id": 2478189,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "50792960"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5892
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942702.790, "dur": 0.437,
    "args": {
      "External id": 2478190,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "51317248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5893
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942703.782, "dur": 0.580,
    "args": {
      "External id": 2478191,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "51841536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5894
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942705.341, "dur": 0.529,
    "args": {
      "External id": 2478192,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "52365824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5895
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942708.275, "dur": 0.212,
    "args": {
      "External id": 2478193,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "52366080"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5896
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942709.117, "dur": 0.743,
    "args": {
      "External id": 2478194,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "53807872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5897
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942710.433, "dur": 0.260,
    "args": {
      "External id": 2478195,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "55249664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5898
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942713.439, "dur": 1.054,
    "args": {
      "External id": 2478196,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "56691456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5899
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942715.278, "dur": 0.446,
    "args": {
      "External id": 2478197,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "56691712"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5900
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942718.304, "dur": 2.007,
    "args": {
      "External id": 2478198,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "57216000"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5901
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942720.973, "dur": 0.414,
    "args": {
      "External id": 2478199,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "57740288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5902
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942722.115, "dur": 0.341,
    "args": {
      "External id": 2478200,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "58264576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5903
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942726.966, "dur": 0.438,
    "args": {
      "External id": 2478201,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "58788864"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5904
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942727.974, "dur": 0.398,
    "args": {
      "External id": 2478202,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "58789120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5905
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942728.988, "dur": 0.311,
    "args": {
      "External id": 2478203,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "60230912"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5906
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942733.102, "dur": 1.069,
    "args": {
      "External id": 2478204,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "61672704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5907
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942734.625, "dur": 0.397,
    "args": {
      "External id": 2478205,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "63114496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5908
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942737.457, "dur": 1.885,
    "args": {
      "External id": 2478206,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "63114752"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5909
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942739.951, "dur": 0.586,
    "args": {
      "External id": 2478207,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "63639040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5910
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942741.171, "dur": 0.335,
    "args": {
      "External id": 2478208,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "64163328"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5911
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942746.072, "dur": 0.159,
    "args": {
      "External id": 2478209,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "64687616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5912
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942746.878, "dur": 0.332,
    "args": {
      "External id": 2478210,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "65211904"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5913
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942747.791, "dur": 0.154,
    "args": {
      "External id": 2478211,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "65212160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5914
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942752.063, "dur": 0.928,
    "args": {
      "External id": 2478212,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "66653952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5915
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942753.564, "dur": 0.155,
    "args": {
      "External id": 2478213,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "68095744"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5916
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942756.180, "dur": 1.692,
    "args": {
      "External id": 2478214,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "69537536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5917
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942758.668, "dur": 0.141,
    "args": {
      "External id": 2478215,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "69537792"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5918
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942759.684, "dur": 0.151,
    "args": {
      "External id": 2478216,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "70062080"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5919
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942764.197, "dur": 0.145,
    "args": {
      "External id": 2478217,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "70586368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5920
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942764.887, "dur": 0.222,
    "args": {
      "External id": 2478218,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "71110656"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5921
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942765.695, "dur": 0.153,
    "args": {
      "External id": 2478219,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "71634944"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5922
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942770.140, "dur": 0.982,
    "args": {
      "External id": 2478220,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "71635200"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5923
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942771.683, "dur": 0.288,
    "args": {
      "External id": 2478221,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "73076992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5924
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942774.473, "dur": 2.199,
    "args": {
      "External id": 2478222,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "74518784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5925
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942777.278, "dur": 0.157,
    "args": {
      "External id": 2478223,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "75960576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5926
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942778.010, "dur": 0.252,
    "args": {
      "External id": 2478224,"Record function id": 0, "Concrete Inputs": ["", "[8192000]", "[1]", "75960832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5927
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942811.139, "dur": 60.697,
    "args": {
      "External id": 2478225,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[], [], []], "Input Dims": [[], [], []], "Ev Idx": 5928
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942942.020, "dur": 384.409,
    "args": {
      "External id": 2478226,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[336611328], [42076416], [], [], []], "Ev Idx": 5929
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514655942976.806, "dur": 344.443,
    "args": {
      "External id": 2478227,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 336611328, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[42076416], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 5930, "In msg nelems": 42076416
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1336754,
    "ts": 1514655943021.267, "dur": 294.218,
    "args": {
      "External id": 2478228,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[42076416]], "Ev Idx": 5931
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514655943350.535, "dur": 1.959,
    "args": {
      "External id": 2478229,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 5932, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out", "pid": 1336754, "tid": 1336754,
    "ts": 1514655943447.892, "dur": 8615.771,
    "args": {
      "External id": 2478230,"Record function id": 0, "Ev Idx": 5933
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655943684.353, "dur": 7.024,
    "args": {
      "External id": 2478231,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[336611328], []], "Ev Idx": 5934
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655943695.771, "dur": 1.134,
    "args": {
      "External id": 2478232,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[65536000], []], "Ev Idx": 5935
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655943698.471, "dur": 0.767,
    "args": {
      "External id": 2478233,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5936
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655943700.871, "dur": 2.140,
    "args": {
      "External id": 2478234,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5937
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655943704.402, "dur": 0.662,
    "args": {
      "External id": 2478235,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5938
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655943708.220, "dur": 0.858,
    "args": {
      "External id": 2478236,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5939
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655943710.705, "dur": 0.675,
    "args": {
      "External id": 2478237,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5940
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655943712.850, "dur": 1.527,
    "args": {
      "External id": 2478238,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5941
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655943717.484, "dur": 1.005,
    "args": {
      "External id": 2478239,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5942
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655943721.795, "dur": 0.780,
    "args": {
      "External id": 2478240,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5943
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655943723.888, "dur": 0.810,
    "args": {
      "External id": 2478241,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5944
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655943726.131, "dur": 1.844,
    "args": {
      "External id": 2478242,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5945
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655943729.420, "dur": 0.632,
    "args": {
      "External id": 2478243,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5946
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655943732.658, "dur": 0.764,
    "args": {
      "External id": 2478244,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5947
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655943734.619, "dur": 1.061,
    "args": {
      "External id": 2478245,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5948
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655943736.956, "dur": 1.736,
    "args": {
      "External id": 2478246,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5949
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655943741.529, "dur": 0.844,
    "args": {
      "External id": 2478247,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5950
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655943745.076, "dur": 0.798,
    "args": {
      "External id": 2478248,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5951
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655943747.262, "dur": 0.841,
    "args": {
      "External id": 2478249,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5952
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655943749.384, "dur": 1.873,
    "args": {
      "External id": 2478250,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5953
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655943752.558, "dur": 0.504,
    "args": {
      "External id": 2478251,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5954
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655943756.185, "dur": 0.687,
    "args": {
      "External id": 2478252,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5955
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655943758.239, "dur": 0.825,
    "args": {
      "External id": 2478253,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5956
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655943760.294, "dur": 1.883,
    "args": {
      "External id": 2478254,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5957
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655943776.887, "dur": 0.925,
    "args": {
      "External id": 2478255,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5958
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655943781.344, "dur": 0.831,
    "args": {
      "External id": 2478256,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5959
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655943783.580, "dur": 0.936,
    "args": {
      "External id": 2478257,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5960
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655943785.765, "dur": 1.714,
    "args": {
      "External id": 2478258,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5961
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655943788.832, "dur": 0.627,
    "args": {
      "External id": 2478259,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5962
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655943792.121, "dur": 0.914,
    "args": {
      "External id": 2478260,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5963
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655943794.224, "dur": 0.604,
    "args": {
      "External id": 2478261,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5964
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655943796.056, "dur": 1.663,
    "args": {
      "External id": 2478262,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5965
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655943800.531, "dur": 0.680,
    "args": {
      "External id": 2478263,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5966
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655943804.103, "dur": 0.636,
    "args": {
      "External id": 2478264,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5967
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655943806.173, "dur": 0.808,
    "args": {
      "External id": 2478265,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5968
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655943808.217, "dur": 2.211,
    "args": {
      "External id": 2478266,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5969
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655943811.644, "dur": 0.563,
    "args": {
      "External id": 2478267,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5970
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655943814.838, "dur": 0.537,
    "args": {
      "External id": 2478268,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5971
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655943816.712, "dur": 0.636,
    "args": {
      "External id": 2478269,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5972
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655943818.759, "dur": 1.669,
    "args": {
      "External id": 2478270,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[65536000], []], "Ev Idx": 5973
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514655943846.984, "dur": 8164.201,
    "args": {
      "External id": 2478271,"Record function id": 0, "Concrete Inputs": ["", "", "1", ""], "Input type": ["c10::BFloat16", "", "Scalar", "TensorList"], "Input Strides": [[42076416, 1], [], [], []], "Input Dims": [[8, 42076416], [], [], []], "Ev Idx": 5974
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514655943869.903, "dur": 8109.230,
    "args": {
      "External id": 2478272,"Record function id": 0, "Concrete Inputs": ["", "", "1", ""], "Input type": ["c10::BFloat16", "", "Scalar", "TensorList"], "Input Strides": [[42076416, 1], [], [], []], "Input Dims": [[8, 42076416], [], [], []], "Ev Idx": 5975
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514655943893.243, "dur": 5.562,
    "args": {
      "External id": 2478273,"Record function id": 0, "Concrete Inputs": ["[4290]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5976
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514655943904.756, "dur": 8035.460,
    "args": {
      "External id": 2478274,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[4290], [], [], [], [], [], [], []], "Ev Idx": 5977
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514655943907.251, "dur": 8032.295,
    "args": {
      "External id": 2478275,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[4290], [], [], [], [], [], []], "Ev Idx": 5978
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655943913.303, "dur": 5.096,
    "args": {
      "External id": 2478276,"Record function id": 0, "Concrete Inputs": ["[4290]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5979
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514655943919.916, "dur": 8016.516,
    "args": {
      "External id": 2478277,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[4290], [4290], []], "Ev Idx": 5980
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336754, "tid": 1336754,
    "ts": 1514655952321.668, "dur": 32.864,
    "args": {
      "External id": 2478278,"Record function id": 0, "Ev Idx": 5981
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 0/0", "pid": 1336754, "tid": 1336754,
    "ts": 1514655952355.759, "dur": 228.228,
    "args": {
      "External id": 2478279,"Record function id": 0, "Ev Idx": 5982
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514655952395.607, "dur": 178.526,
    "args": {
      "External id": 2478280,"Sequence number": 24740420, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "long int"], "Input Strides": [[2048, 1], [4096, 1]], "Input Dims": [[32000, 2048], [16, 4096]], "Ev Idx": 5983
    }
  },
  {
    "ph": "s", "id": 228, "pid": 1336754, "tid": 1336754, "ts": 1514655952395.607,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_embedding_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514655952466.353, "dur": 53.435,
    "args": {
      "External id": 2478281,"kernel_hash": "c5wemnzn2hvtwjoj3icw7tagar3wjyj7c7i5vhvuszrcw63rsbhp", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/5w/c5wemnzn2hvtwjoj3icw7tagar3wjyj7c7i5vhvuszrcw63rsbhp.py", "kernel_backend": "triton", "Input type": ["long int", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[4096, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096], [32000, 2048], [16, 4096, 2048], []], "Ev Idx": 5984
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336754, "tid": 1336754,
    "ts": 1514655952646.069, "dur": 55.773,
    "args": {
      "External id": 2478282,"Record function id": 0, "Ev Idx": 5985
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.0)", "pid": 1336754, "tid": 1336754,
    "ts": 1514655952712.639, "dur": 8026.061,
    "args": {
      "External id": 2478283,"Record function id": 0, "Ev Idx": 5986
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.0)", "pid": 1336754, "tid": 1336754,
    "ts": 1514655952720.722, "dur": 850.915,
    "args": {
      "External id": 2478284,"Record function id": 0, "Ev Idx": 5987
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514655952798.402, "dur": 10.340,
    "args": {
      "External id": 2478285,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5988
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514655952821.941, "dur": 35.907,
    "args": {
      "External id": 2478286,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 5989
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655952830.301, "dur": 2.402,
    "args": {
      "External id": 2478287,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5990
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655952834.279, "dur": 0.433,
    "args": {
      "External id": 2478288,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5991
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655952835.359, "dur": 2.117,
    "args": {
      "External id": 2478289,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5992
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655952840.223, "dur": 0.392,
    "args": {
      "External id": 2478290,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5993
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655952841.343, "dur": 0.150,
    "args": {
      "External id": 2478291,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5994
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655952845.886, "dur": 0.216,
    "args": {
      "External id": 2478292,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5995
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655952846.755, "dur": 0.141,
    "args": {
      "External id": 2478293,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5996
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655952847.483, "dur": 1.120,
    "args": {
      "External id": 2478294,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5997
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655952851.362, "dur": 0.444,
    "args": {
      "External id": 2478295,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5998
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514655952868.219, "dur": 39.586,
    "args": {
      "External id": 2478296,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 5999
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1336754,
    "ts": 1514655952939.395, "dur": 191.262,
    "args": {
      "External id": 2478297,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "1", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 6000
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514655952950.743, "dur": 3.716,
    "args": {
      "External id": 2478298,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6001
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1336754,
    "ts": 1514655952959.581, "dur": 10.693,
    "args": {
      "External id": 2478299,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6002
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514655952963.428, "dur": 6.463,
    "args": {
      "External id": 2478300,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "12846080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 6003
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655952967.882, "dur": 0.704,
    "args": {
      "External id": 2478301,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6004
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514655952976.006, "dur": 100.342,
    "args": {
      "External id": 2478302,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6005
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655952977.574, "dur": 0.587,
    "args": {
      "External id": 2478303,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6006
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655953051.129, "dur": 1.367,
    "args": {
      "External id": 2478304,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6423296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6007
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655953056.826, "dur": 0.390,
    "args": {
      "External id": 2478305,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6947584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6008
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655953059.783, "dur": 0.462,
    "args": {
      "External id": 2478306,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7471872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6009
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655953061.272, "dur": 1.957,
    "args": {
      "External id": 2478307,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7996160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6010
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655953064.093, "dur": 0.373,
    "args": {
      "External id": 2478308,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8520448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6011
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655953067.158, "dur": 0.432,
    "args": {
      "External id": 2478309,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "8520704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6012
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655953070.202, "dur": 0.148,
    "args": {
      "External id": 2478310,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "9962496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6013
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655953070.841, "dur": 0.473,
    "args": {
      "External id": 2478311,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11404288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6014
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514655953092.522, "dur": 29.120,
    "args": {
      "External id": 2478312,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6015
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1336754,
    "ts": 1514655953185.917, "dur": 282.995,
    "args": {
      "External id": 2478313,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 6016
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514655953214.311, "dur": 250.105,
    "args": {
      "External id": 2478314,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 6017, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1336754,
    "ts": 1514655953223.934, "dur": 235.742,
    "args": {
      "External id": 2478315,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 6018
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514655953488.411, "dur": 2.084,
    "args": {
      "External id": 2478316,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 6019, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.0)", "pid": 1336754, "tid": 1336754,
    "ts": 1514655953595.868, "dur": 6907.552,
    "args": {
      "External id": 2478317,"Record function id": 0, "Ev Idx": 6020
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655953694.611, "dur": 6.229,
    "args": {
      "External id": 2478318,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 6021
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655953704.357, "dur": 1.118,
    "args": {
      "External id": 2478319,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6022
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655953706.998, "dur": 0.831,
    "args": {
      "External id": 2478320,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6023
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655953710.880, "dur": 2.050,
    "args": {
      "External id": 2478321,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6024
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655953714.656, "dur": 0.747,
    "args": {
      "External id": 2478322,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6025
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655953716.680, "dur": 0.598,
    "args": {
      "External id": 2478323,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6026
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655953718.653, "dur": 0.949,
    "args": {
      "External id": 2478324,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6027
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655953724.312, "dur": 1.534,
    "args": {
      "External id": 2478325,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6028
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655953727.145, "dur": 0.748,
    "args": {
      "External id": 2478326,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6029
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655953729.106, "dur": 0.760,
    "args": {
      "External id": 2478327,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6030
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514655953745.921, "dur": 6722.221,
    "args": {
      "External id": 2478328,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6031
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514655953761.516, "dur": 6700.003,
    "args": {
      "External id": 2478329,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6032
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514655953781.003, "dur": 13.694,
    "args": {
      "External id": 2478330,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6033
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514655953800.355, "dur": 6629.560,
    "args": {
      "External id": 2478331,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 6034
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514655953802.766, "dur": 6626.615,
    "args": {
      "External id": 2478332,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 6035
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655953808.058, "dur": 7.257,
    "args": {
      "External id": 2478333,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6036
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514655953817.339, "dur": 6609.171,
    "args": {
      "External id": 2478334,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 6037
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514655960682.929, "dur": 31.407,
    "args": {
      "External id": 2478335,"Sequence number": 24740421, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 6038
    }
  },
  {
    "ph": "s", "id": 227, "pid": 1336754, "tid": 1336754, "ts": 1514655960682.929,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336754, "tid": 1336754,
    "ts": 1514655960700.531, "dur": 9.062,
    "args": {
      "External id": 2478336,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 6039
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655960705.193, "dur": 4.061,
    "args": {
      "External id": 2478337,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 6040
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336754, "tid": 1336754,
    "ts": 1514655960775.266, "dur": 91.829,
    "args": {
      "External id": 2478338,"Record function id": 0, "Ev Idx": 6041
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336754, "tid": 1336754,
    "ts": 1514655960868.434, "dur": 1196.601,
    "args": {
      "External id": 2478339,"Record function id": 0, "Ev Idx": 6042
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514655960908.856, "dur": 1140.797,
    "args": {
      "External id": 2478340,"Sequence number": 24740422, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 6043
    }
  },
  {
    "ph": "s", "id": 226, "pid": 1336754, "tid": 1336754, "ts": 1514655960908.856,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514655960973.445, "dur": 93.554,
    "args": {
      "External id": 2478341,"kernel_hash": "caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/ao/caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6044
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514655961083.222, "dur": 112.906,
    "args": {
      "External id": 2478342,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6045
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514655961207.742, "dur": 39.610,
    "args": {
      "External id": 2478343,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6046
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514655961254.983, "dur": 31.473,
    "args": {
      "External id": 2478344,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6047
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514655961313.727, "dur": 29.010,
    "args": {
      "External id": 2478345,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6048
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514655961361.556, "dur": 16.281,
    "args": {
      "External id": 2478346,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6049
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336754, "tid": 1336754,
    "ts": 1514655961398.800, "dur": 159.768,
    "args": {
      "External id": 2478347,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 6050
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514655961452.314, "dur": 10.357,
    "args": {
      "External id": 2478348,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 6051
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655961456.622, "dur": 5.303,
    "args": {
      "External id": 2478349,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6052
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514655961465.003, "dur": 4.077,
    "args": {
      "External id": 2478350,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6053
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514655961472.237, "dur": 0.855,
    "args": {
      "External id": 2478351,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6054
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514655961475.319, "dur": 3.313,
    "args": {
      "External id": 2478352,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6055
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514655961571.439, "dur": 55.075,
    "args": {
      "External id": 2478353,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6056
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336754, "tid": 1336754,
    "ts": 1514655961660.158, "dur": 31.153,
    "args": {
      "External id": 2478354,"kernel_hash": "cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/wx/cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6057
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514655961699.045, "dur": 43.397,
    "args": {
      "External id": 2478355,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6058
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514655961753.344, "dur": 35.706,
    "args": {
      "External id": 2478356,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6059
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514655961809.729, "dur": 26.709,
    "args": {
      "External id": 2478357,"kernel_hash": "c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2f/c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 6060
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514655961842.335, "dur": 35.722,
    "args": {
      "External id": 2478358,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 6061
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514655961903.449, "dur": 20.141,
    "args": {
      "External id": 2478359,"kernel_hash": "cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/gi/cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 6062
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.0)", "pid": 1336754, "tid": 1336754,
    "ts": 1514655962127.853, "dur": 84.719,
    "args": {
      "External id": 2478360,"Record function id": 0, "Ev Idx": 6063
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336754, "tid": 1336754,
    "ts": 1514655962282.858, "dur": 46.068,
    "args": {
      "External id": 2478361,"Record function id": 0, "Ev Idx": 6064
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.1)", "pid": 1336754, "tid": 1336754,
    "ts": 1514655962338.031, "dur": 18909.862,
    "args": {
      "External id": 2478362,"Record function id": 0, "Ev Idx": 6065
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.1)", "pid": 1336754, "tid": 1336754,
    "ts": 1514655962348.355, "dur": 933.218,
    "args": {
      "External id": 2478363,"Record function id": 0, "Ev Idx": 6066
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514655962429.906, "dur": 8.670,
    "args": {
      "External id": 2478364,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6067
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514655962451.280, "dur": 39.762,
    "args": {
      "External id": 2478365,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6068
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655962457.968, "dur": 3.594,
    "args": {
      "External id": 2478366,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6069
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655962463.100, "dur": 0.653,
    "args": {
      "External id": 2478367,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6070
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655962466.270, "dur": 0.466,
    "args": {
      "External id": 2478368,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6071
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655962469.077, "dur": 0.268,
    "args": {
      "External id": 2478369,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6072
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655962470.116, "dur": 2.568,
    "args": {
      "External id": 2478370,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6073
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655962475.470, "dur": 0.180,
    "args": {
      "External id": 2478371,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6074
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655962477.797, "dur": 0.484,
    "args": {
      "External id": 2478372,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6075
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655962478.934, "dur": 0.453,
    "args": {
      "External id": 2478373,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6076
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655962482.176, "dur": 2.304,
    "args": {
      "External id": 2478374,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6077
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514655962501.450, "dur": 66.976,
    "args": {
      "External id": 2478375,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6078
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1336754,
    "ts": 1514655962603.613, "dur": 121.979,
    "args": {
      "External id": 2478376,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "1", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 6079
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514655962616.211, "dur": 4.994,
    "args": {
      "External id": 2478377,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6080
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1336754,
    "ts": 1514655962627.065, "dur": 10.361,
    "args": {
      "External id": 2478378,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6081
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514655962631.475, "dur": 5.530,
    "args": {
      "External id": 2478379,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "12846080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 6082
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655962634.469, "dur": 1.046,
    "args": {
      "External id": 2478380,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6083
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514655962643.834, "dur": 33.275,
    "args": {
      "External id": 2478381,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6084
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655962647.529, "dur": 0.289,
    "args": {
      "External id": 2478382,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6085
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655962648.957, "dur": 1.764,
    "args": {
      "External id": 2478383,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6423296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6086
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655962651.347, "dur": 2.112,
    "args": {
      "External id": 2478384,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6947584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6087
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655962655.546, "dur": 0.394,
    "args": {
      "External id": 2478385,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7471872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6088
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655962656.683, "dur": 0.542,
    "args": {
      "External id": 2478386,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7996160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6089
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655962659.786, "dur": 0.233,
    "args": {
      "External id": 2478387,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8520448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6090
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655962662.522, "dur": 0.177,
    "args": {
      "External id": 2478388,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "8520704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6091
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655962663.291, "dur": 0.310,
    "args": {
      "External id": 2478389,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "9962496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6092
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655962668.007, "dur": 0.236,
    "args": {
      "External id": 2478390,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11404288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6093
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514655962689.579, "dur": 26.732,
    "args": {
      "External id": 2478391,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6094
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1336754,
    "ts": 1514655962779.301, "dur": 409.730,
    "args": {
      "External id": 2478392,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 6095
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514655962809.360, "dur": 374.022,
    "args": {
      "External id": 2478393,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 6096, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1336754,
    "ts": 1514655962818.753, "dur": 358.655,
    "args": {
      "External id": 2478394,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 6097
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514655963211.654, "dur": 2.499,
    "args": {
      "External id": 2478395,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 6098, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.1)", "pid": 1336754, "tid": 1336754,
    "ts": 1514655963303.050, "dur": 17758.004,
    "args": {
      "External id": 2478396,"Record function id": 0, "Ev Idx": 6099
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655963399.085, "dur": 6.195,
    "args": {
      "External id": 2478397,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 6100
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655963408.711, "dur": 1.096,
    "args": {
      "External id": 2478398,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6101
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655963411.511, "dur": 0.940,
    "args": {
      "External id": 2478399,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6102
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655963414.106, "dur": 1.116,
    "args": {
      "External id": 2478400,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6103
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655963416.647, "dur": 0.980,
    "args": {
      "External id": 2478401,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6104
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655963420.958, "dur": 0.792,
    "args": {
      "External id": 2478402,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6105
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655963424.707, "dur": 0.753,
    "args": {
      "External id": 2478403,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6106
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655963426.683, "dur": 3.609,
    "args": {
      "External id": 2478404,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6107
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655963431.811, "dur": 0.814,
    "args": {
      "External id": 2478405,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6108
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655963435.742, "dur": 0.976,
    "args": {
      "External id": 2478406,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6109
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514655963453.115, "dur": 17570.165,
    "args": {
      "External id": 2478407,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6110
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514655963469.574, "dur": 17546.598,
    "args": {
      "External id": 2478408,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6111
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514655963490.148, "dur": 12.851,
    "args": {
      "External id": 2478409,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6112
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514655963506.222, "dur": 17452.600,
    "args": {
      "External id": 2478410,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 6113
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514655963508.630, "dur": 17449.716,
    "args": {
      "External id": 2478411,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 6114
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655963513.443, "dur": 4.769,
    "args": {
      "External id": 2478412,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6115
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514655963519.658, "dur": 17435.814,
    "args": {
      "External id": 2478413,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 6116
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514655981195.258, "dur": 29.380,
    "args": {
      "External id": 2478414,"Sequence number": 24740423, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 6117
    }
  },
  {
    "ph": "s", "id": 225, "pid": 1336754, "tid": 1336754, "ts": 1514655981195.258,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336754, "tid": 1336754,
    "ts": 1514655981210.603, "dur": 9.167,
    "args": {
      "External id": 2478415,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 6118
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655981215.516, "dur": 4.045,
    "args": {
      "External id": 2478416,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 6119
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336754, "tid": 1336754,
    "ts": 1514655981285.440, "dur": 78.949,
    "args": {
      "External id": 2478417,"Record function id": 0, "Ev Idx": 6120
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336754, "tid": 1336754,
    "ts": 1514655981365.816, "dur": 1061.379,
    "args": {
      "External id": 2478418,"Record function id": 0, "Ev Idx": 6121
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514655981404.029, "dur": 1009.984,
    "args": {
      "External id": 2478419,"Sequence number": 24740424, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 6122
    }
  },
  {
    "ph": "s", "id": 224, "pid": 1336754, "tid": 1336754, "ts": 1514655981404.029,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514655981480.791, "dur": 42.219,
    "args": {
      "External id": 2478420,"kernel_hash": "caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/ao/caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6123
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514655981557.571, "dur": 102.766,
    "args": {
      "External id": 2478421,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6124
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514655981669.100, "dur": 38.833,
    "args": {
      "External id": 2478422,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6125
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514655981713.751, "dur": 31.734,
    "args": {
      "External id": 2478423,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6126
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514655981773.894, "dur": 24.319,
    "args": {
      "External id": 2478424,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6127
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514655981812.565, "dur": 17.169,
    "args": {
      "External id": 2478425,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6128
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336754, "tid": 1336754,
    "ts": 1514655981844.649, "dur": 122.622,
    "args": {
      "External id": 2478426,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 6129
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514655981892.200, "dur": 11.602,
    "args": {
      "External id": 2478427,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 6130
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655981896.433, "dur": 6.697,
    "args": {
      "External id": 2478428,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6131
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514655981907.676, "dur": 4.641,
    "args": {
      "External id": 2478429,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6132
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514655981913.391, "dur": 2.695,
    "args": {
      "External id": 2478430,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6133
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514655981918.205, "dur": 2.795,
    "args": {
      "External id": 2478431,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6134
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514655981977.537, "dur": 89.520,
    "args": {
      "External id": 2478432,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6135
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336754, "tid": 1336754,
    "ts": 1514655982102.195, "dur": 28.273,
    "args": {
      "External id": 2478433,"kernel_hash": "cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/wx/cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6136
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514655982139.368, "dur": 45.284,
    "args": {
      "External id": 2478434,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6137
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514655982190.649, "dur": 35.286,
    "args": {
      "External id": 2478435,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6138
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514655982251.011, "dur": 26.329,
    "args": {
      "External id": 2478436,"kernel_hash": "c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2f/c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 6139
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514655982282.745, "dur": 33.519,
    "args": {
      "External id": 2478437,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 6140
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514655982333.503, "dur": 19.131,
    "args": {
      "External id": 2478438,"kernel_hash": "cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/gi/cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 6141
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.1)", "pid": 1336754, "tid": 1336754,
    "ts": 1514655982488.982, "dur": 93.445,
    "args": {
      "External id": 2478439,"Record function id": 0, "Ev Idx": 6142
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336754, "tid": 1336754,
    "ts": 1514655982658.198, "dur": 47.088,
    "args": {
      "External id": 2478440,"Record function id": 0, "Ev Idx": 6143
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.2)", "pid": 1336754, "tid": 1336754,
    "ts": 1514655982714.863, "dur": 18012.587,
    "args": {
      "External id": 2478441,"Record function id": 0, "Ev Idx": 6144
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.2)", "pid": 1336754, "tid": 1336754,
    "ts": 1514655982724.730, "dur": 920.904,
    "args": {
      "External id": 2478442,"Record function id": 0, "Ev Idx": 6145
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514655982805.014, "dur": 8.080,
    "args": {
      "External id": 2478443,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6146
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514655982828.058, "dur": 35.945,
    "args": {
      "External id": 2478444,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6147
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655982832.719, "dur": 2.232,
    "args": {
      "External id": 2478445,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6148
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655982838.690, "dur": 0.394,
    "args": {
      "External id": 2478446,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6149
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655982841.888, "dur": 0.329,
    "args": {
      "External id": 2478447,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6150
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655982842.896, "dur": 0.449,
    "args": {
      "External id": 2478448,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6151
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655982845.528, "dur": 0.371,
    "args": {
      "External id": 2478449,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6152
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655982848.243, "dur": 0.660,
    "args": {
      "External id": 2478450,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6153
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655982849.476, "dur": 3.427,
    "args": {
      "External id": 2478451,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6154
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655982855.839, "dur": 0.357,
    "args": {
      "External id": 2478452,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6155
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655982856.991, "dur": 0.723,
    "args": {
      "External id": 2478453,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6156
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514655982873.922, "dur": 42.958,
    "args": {
      "External id": 2478454,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6157
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1336754,
    "ts": 1514655982949.491, "dur": 154.931,
    "args": {
      "External id": 2478455,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "1", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 6158
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514655982961.292, "dur": 3.399,
    "args": {
      "External id": 2478456,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6159
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1336754,
    "ts": 1514655982969.550, "dur": 8.947,
    "args": {
      "External id": 2478457,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6160
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514655982973.631, "dur": 4.473,
    "args": {
      "External id": 2478458,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "12846080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 6161
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655982976.560, "dur": 0.431,
    "args": {
      "External id": 2478459,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6162
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514655983024.962, "dur": 32.052,
    "args": {
      "External id": 2478460,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6163
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655983028.783, "dur": 2.159,
    "args": {
      "External id": 2478461,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6164
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655983032.228, "dur": 0.340,
    "args": {
      "External id": 2478462,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6423296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6165
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655983033.268, "dur": 0.438,
    "args": {
      "External id": 2478463,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6947584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6166
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655983037.794, "dur": 1.997,
    "args": {
      "External id": 2478464,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7471872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6167
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655983040.380, "dur": 0.499,
    "args": {
      "External id": 2478465,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7996160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6168
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655983043.266, "dur": 0.353,
    "args": {
      "External id": 2478466,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8520448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6169
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655983046.039, "dur": 0.424,
    "args": {
      "External id": 2478467,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "8520704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6170
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655983047.217, "dur": 0.242,
    "args": {
      "External id": 2478468,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "9962496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6171
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655983049.859, "dur": 2.228,
    "args": {
      "External id": 2478469,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11404288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6172
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514655983068.079, "dur": 28.019,
    "args": {
      "External id": 2478470,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6173
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1336754,
    "ts": 1514655983159.831, "dur": 386.732,
    "args": {
      "External id": 2478471,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 6174
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514655983190.253, "dur": 332.617,
    "args": {
      "External id": 2478472,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 6175, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1336754,
    "ts": 1514655983201.582, "dur": 316.279,
    "args": {
      "External id": 2478473,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 6176
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514655983572.275, "dur": 3.080,
    "args": {
      "External id": 2478474,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 6177, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.2)", "pid": 1336754, "tid": 1336754,
    "ts": 1514655983666.207, "dur": 16848.596,
    "args": {
      "External id": 2478475,"Record function id": 0, "Ev Idx": 6178
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655983767.058, "dur": 5.974,
    "args": {
      "External id": 2478476,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 6179
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655983776.298, "dur": 1.085,
    "args": {
      "External id": 2478477,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6180
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655983778.981, "dur": 2.430,
    "args": {
      "External id": 2478478,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6181
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655983783.127, "dur": 1.170,
    "args": {
      "External id": 2478479,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6182
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655983785.656, "dur": 0.904,
    "args": {
      "External id": 2478480,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6183
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655983788.165, "dur": 0.935,
    "args": {
      "External id": 2478481,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6184
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655983792.505, "dur": 1.048,
    "args": {
      "External id": 2478482,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6185
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655983795.274, "dur": 2.031,
    "args": {
      "External id": 2478483,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6186
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655983798.584, "dur": 0.853,
    "args": {
      "External id": 2478484,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6187
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514655983800.930, "dur": 0.679,
    "args": {
      "External id": 2478485,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6188
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514655983821.658, "dur": 16654.942,
    "args": {
      "External id": 2478486,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6189
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514655983835.773, "dur": 16634.114,
    "args": {
      "External id": 2478487,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6190
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514655983856.013, "dur": 14.672,
    "args": {
      "External id": 2478488,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6191
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514655983873.467, "dur": 16564.250,
    "args": {
      "External id": 2478489,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 6192
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514655983876.163, "dur": 16560.991,
    "args": {
      "External id": 2478490,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 6193
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514655983881.326, "dur": 4.722,
    "args": {
      "External id": 2478491,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6194
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514655983887.562, "dur": 16546.449,
    "args": {
      "External id": 2478492,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 6195
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656000673.895, "dur": 29.544,
    "args": {
      "External id": 2478493,"Sequence number": 24740425, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 6196
    }
  },
  {
    "ph": "s", "id": 223, "pid": 1336754, "tid": 1336754, "ts": 1514656000673.895,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336754, "tid": 1336754,
    "ts": 1514656000691.903, "dur": 6.833,
    "args": {
      "External id": 2478494,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 6197
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656000694.466, "dur": 3.922,
    "args": {
      "External id": 2478495,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 6198
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336754, "tid": 1336754,
    "ts": 1514656000766.573, "dur": 72.974,
    "args": {
      "External id": 2478496,"Record function id": 0, "Ev Idx": 6199
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656000840.903, "dur": 1085.718,
    "args": {
      "External id": 2478497,"Record function id": 0, "Ev Idx": 6200
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656000880.994, "dur": 1032.260,
    "args": {
      "External id": 2478498,"Sequence number": 24740426, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 6201
    }
  },
  {
    "ph": "s", "id": 222, "pid": 1336754, "tid": 1336754, "ts": 1514656000880.994,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514656000943.735, "dur": 89.830,
    "args": {
      "External id": 2478499,"kernel_hash": "caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/ao/caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6202
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656001049.652, "dur": 104.963,
    "args": {
      "External id": 2478500,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6203
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656001163.859, "dur": 41.675,
    "args": {
      "External id": 2478501,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6204
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656001214.539, "dur": 31.899,
    "args": {
      "External id": 2478502,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6205
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656001270.382, "dur": 24.693,
    "args": {
      "External id": 2478503,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6206
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656001310.956, "dur": 15.665,
    "args": {
      "External id": 2478504,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6207
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336754, "tid": 1336754,
    "ts": 1514656001344.382, "dur": 122.148,
    "args": {
      "External id": 2478505,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 6208
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656001391.375, "dur": 11.549,
    "args": {
      "External id": 2478506,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 6209
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656001397.353, "dur": 4.752,
    "args": {
      "External id": 2478507,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6210
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656001405.226, "dur": 5.703,
    "args": {
      "External id": 2478508,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6211
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656001412.050, "dur": 1.090,
    "args": {
      "External id": 2478509,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6212
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656001415.328, "dur": 3.516,
    "args": {
      "External id": 2478510,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6213
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656001476.423, "dur": 45.556,
    "args": {
      "External id": 2478511,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6214
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336754, "tid": 1336754,
    "ts": 1514656001589.664, "dur": 31.341,
    "args": {
      "External id": 2478512,"kernel_hash": "cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/wx/cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6215
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656001630.427, "dur": 45.417,
    "args": {
      "External id": 2478513,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6216
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656001683.820, "dur": 35.934,
    "args": {
      "External id": 2478514,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6217
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514656001739.610, "dur": 24.225,
    "args": {
      "External id": 2478515,"kernel_hash": "c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2f/c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 6218
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656001769.109, "dur": 34.917,
    "args": {
      "External id": 2478516,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 6219
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656001825.779, "dur": 17.205,
    "args": {
      "External id": 2478517,"kernel_hash": "cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/gi/cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 6220
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.2)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656002042.108, "dur": 76.318,
    "args": {
      "External id": 2478518,"Record function id": 0, "Ev Idx": 6221
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656002193.208, "dur": 45.666,
    "args": {
      "External id": 2478519,"Record function id": 0, "Ev Idx": 6222
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.3)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656002248.709, "dur": 18948.910,
    "args": {
      "External id": 2478520,"Record function id": 0, "Ev Idx": 6223
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.3)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656002258.450, "dur": 1021.178,
    "args": {
      "External id": 2478521,"Record function id": 0, "Ev Idx": 6224
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656002337.598, "dur": 8.546,
    "args": {
      "External id": 2478522,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6225
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514656002358.905, "dur": 40.145,
    "args": {
      "External id": 2478523,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6226
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656002363.829, "dur": 2.420,
    "args": {
      "External id": 2478524,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6227
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656002371.735, "dur": 0.396,
    "args": {
      "External id": 2478525,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6228
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656002373.184, "dur": 0.441,
    "args": {
      "External id": 2478526,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6229
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656002378.521, "dur": 0.604,
    "args": {
      "External id": 2478527,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6230
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656002383.836, "dur": 0.414,
    "args": {
      "External id": 2478528,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6231
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656002385.186, "dur": 0.412,
    "args": {
      "External id": 2478529,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6232
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656002387.478, "dur": 2.873,
    "args": {
      "External id": 2478530,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6233
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656002390.979, "dur": 0.183,
    "args": {
      "External id": 2478531,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6234
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656002391.802, "dur": 0.389,
    "args": {
      "External id": 2478532,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6235
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656002411.719, "dur": 45.380,
    "args": {
      "External id": 2478533,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6236
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1336754,
    "ts": 1514656002488.498, "dur": 154.888,
    "args": {
      "External id": 2478534,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "1", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 6237
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656002500.108, "dur": 3.406,
    "args": {
      "External id": 2478535,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6238
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1336754,
    "ts": 1514656002508.150, "dur": 9.439,
    "args": {
      "External id": 2478536,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6239
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656002512.507, "dur": 4.697,
    "args": {
      "External id": 2478537,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "12846080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 6240
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656002515.510, "dur": 0.529,
    "args": {
      "External id": 2478538,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6241
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514656002525.021, "dur": 58.661,
    "args": {
      "External id": 2478539,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6242
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656002553.955, "dur": 2.515,
    "args": {
      "External id": 2478540,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6243
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656002558.393, "dur": 0.447,
    "args": {
      "External id": 2478541,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6423296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6244
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656002561.173, "dur": 0.384,
    "args": {
      "External id": 2478542,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6947584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6245
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656002564.103, "dur": 2.016,
    "args": {
      "External id": 2478543,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7471872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6246
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656002568.587, "dur": 0.491,
    "args": {
      "External id": 2478544,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7996160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6247
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656002569.791, "dur": 0.639,
    "args": {
      "External id": 2478545,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8520448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6248
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656002572.988, "dur": 0.334,
    "args": {
      "External id": 2478546,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "8520704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6249
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656002575.348, "dur": 0.258,
    "args": {
      "External id": 2478547,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "9962496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6250
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656002576.448, "dur": 1.861,
    "args": {
      "External id": 2478548,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11404288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6251
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656002605.344, "dur": 29.402,
    "args": {
      "External id": 2478549,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6252
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656002694.957, "dur": 483.287,
    "args": {
      "External id": 2478550,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 6253
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514656002726.460, "dur": 446.134,
    "args": {
      "External id": 2478551,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 6254, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1336754,
    "ts": 1514656002739.284, "dur": 425.296,
    "args": {
      "External id": 2478552,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 6255
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514656003204.882, "dur": 2.613,
    "args": {
      "External id": 2478553,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 6256, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.3)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656003299.934, "dur": 17675.353,
    "args": {
      "External id": 2478554,"Record function id": 0, "Ev Idx": 6257
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656003395.840, "dur": 6.078,
    "args": {
      "External id": 2478555,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 6258
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656003405.483, "dur": 0.969,
    "args": {
      "External id": 2478556,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6259
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656003408.286, "dur": 2.662,
    "args": {
      "External id": 2478557,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6260
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656003412.554, "dur": 1.084,
    "args": {
      "External id": 2478558,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6261
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656003414.915, "dur": 1.143,
    "args": {
      "External id": 2478559,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6262
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656003417.409, "dur": 1.120,
    "args": {
      "External id": 2478560,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6263
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656003421.598, "dur": 0.847,
    "args": {
      "External id": 2478561,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6264
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656003423.965, "dur": 1.989,
    "args": {
      "External id": 2478562,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6265
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656003427.430, "dur": 0.628,
    "args": {
      "External id": 2478563,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6266
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656003429.415, "dur": 0.900,
    "args": {
      "External id": 2478564,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6267
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656003448.880, "dur": 17488.272,
    "args": {
      "External id": 2478565,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6268
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656003464.645, "dur": 17465.207,
    "args": {
      "External id": 2478566,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6269
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656003484.431, "dur": 14.162,
    "args": {
      "External id": 2478567,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6270
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514656003501.689, "dur": 17396.206,
    "args": {
      "External id": 2478568,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 6271
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656003504.207, "dur": 17393.160,
    "args": {
      "External id": 2478569,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 6272
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656003510.617, "dur": 5.907,
    "args": {
      "External id": 2478570,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6273
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656003518.159, "dur": 17376.130,
    "args": {
      "External id": 2478571,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 6274
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656021128.944, "dur": 44.553,
    "args": {
      "External id": 2478572,"Sequence number": 24740427, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 6275
    }
  },
  {
    "ph": "s", "id": 221, "pid": 1336754, "tid": 1336754, "ts": 1514656021128.944,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336754, "tid": 1336754,
    "ts": 1514656021161.506, "dur": 7.304,
    "args": {
      "External id": 2478573,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 6276
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656021164.440, "dur": 4.050,
    "args": {
      "External id": 2478574,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 6277
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336754, "tid": 1336754,
    "ts": 1514656021233.807, "dur": 77.281,
    "args": {
      "External id": 2478575,"Record function id": 0, "Ev Idx": 6278
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656021314.398, "dur": 1096.975,
    "args": {
      "External id": 2478576,"Record function id": 0, "Ev Idx": 6279
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656021351.271, "dur": 1047.082,
    "args": {
      "External id": 2478577,"Sequence number": 24740428, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 6280
    }
  },
  {
    "ph": "s", "id": 220, "pid": 1336754, "tid": 1336754, "ts": 1514656021351.271,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514656021413.645, "dur": 42.528,
    "args": {
      "External id": 2478578,"kernel_hash": "caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/ao/caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6281
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656021468.235, "dur": 119.954,
    "args": {
      "External id": 2478579,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6282
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656021605.034, "dur": 44.156,
    "args": {
      "External id": 2478580,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6283
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656021655.225, "dur": 31.221,
    "args": {
      "External id": 2478581,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6284
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656021716.143, "dur": 26.780,
    "args": {
      "External id": 2478582,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6285
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656021759.793, "dur": 15.845,
    "args": {
      "External id": 2478583,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6286
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336754, "tid": 1336754,
    "ts": 1514656021792.793, "dur": 121.601,
    "args": {
      "External id": 2478584,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 6287
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656021840.890, "dur": 10.094,
    "args": {
      "External id": 2478585,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 6288
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656021845.349, "dur": 4.964,
    "args": {
      "External id": 2478586,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6289
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656021853.165, "dur": 4.603,
    "args": {
      "External id": 2478587,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6290
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656021858.977, "dur": 0.786,
    "args": {
      "External id": 2478588,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6291
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656021863.598, "dur": 3.743,
    "args": {
      "External id": 2478589,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6292
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656021925.759, "dur": 48.186,
    "args": {
      "External id": 2478590,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6293
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336754, "tid": 1336754,
    "ts": 1514656022046.513, "dur": 30.603,
    "args": {
      "External id": 2478591,"kernel_hash": "cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/wx/cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6294
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656022087.889, "dur": 47.916,
    "args": {
      "External id": 2478592,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6295
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656022165.018, "dur": 37.241,
    "args": {
      "External id": 2478593,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6296
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514656022228.813, "dur": 25.315,
    "args": {
      "External id": 2478594,"kernel_hash": "c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2f/c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 6297
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656022260.196, "dur": 34.369,
    "args": {
      "External id": 2478595,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 6298
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656022310.894, "dur": 24.331,
    "args": {
      "External id": 2478596,"kernel_hash": "cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/gi/cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 6299
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.3)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656022474.360, "dur": 96.038,
    "args": {
      "External id": 2478597,"Record function id": 0, "Ev Idx": 6300
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656022644.717, "dur": 46.870,
    "args": {
      "External id": 2478598,"Record function id": 0, "Ev Idx": 6301
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.4)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656022699.920, "dur": 17800.227,
    "args": {
      "External id": 2478599,"Record function id": 0, "Ev Idx": 6302
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.4)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656022709.614, "dur": 892.309,
    "args": {
      "External id": 2478600,"Record function id": 0, "Ev Idx": 6303
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656022790.378, "dur": 8.332,
    "args": {
      "External id": 2478601,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6304
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514656022811.520, "dur": 37.944,
    "args": {
      "External id": 2478602,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6305
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656022817.677, "dur": 2.347,
    "args": {
      "External id": 2478603,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6306
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656022823.700, "dur": 0.429,
    "args": {
      "External id": 2478604,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6307
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656022824.762, "dur": 0.331,
    "args": {
      "External id": 2478605,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6308
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656022827.404, "dur": 0.291,
    "args": {
      "External id": 2478606,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6309
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656022830.328, "dur": 0.555,
    "args": {
      "External id": 2478607,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6310
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656022832.755, "dur": 0.490,
    "args": {
      "External id": 2478608,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6311
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656022833.871, "dur": 3.836,
    "args": {
      "External id": 2478609,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6312
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656022838.568, "dur": 0.376,
    "args": {
      "External id": 2478610,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6313
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656022841.441, "dur": 0.566,
    "args": {
      "External id": 2478611,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6314
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656022859.507, "dur": 42.738,
    "args": {
      "External id": 2478612,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6315
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1336754,
    "ts": 1514656022933.250, "dur": 157.344,
    "args": {
      "External id": 2478613,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "1", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 6316
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656022946.157, "dur": 3.871,
    "args": {
      "External id": 2478614,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6317
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1336754,
    "ts": 1514656022954.682, "dur": 10.367,
    "args": {
      "External id": 2478615,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6318
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656022958.800, "dur": 5.849,
    "args": {
      "External id": 2478616,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "12846080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 6319
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656022962.822, "dur": 0.624,
    "args": {
      "External id": 2478617,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6320
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514656022971.255, "dur": 68.363,
    "args": {
      "External id": 2478618,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6321
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656022972.826, "dur": 2.361,
    "args": {
      "External id": 2478619,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6322
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656022977.700, "dur": 0.634,
    "args": {
      "External id": 2478620,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6423296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6323
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656022979.057, "dur": 0.569,
    "args": {
      "External id": 2478621,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6947584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6324
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656023020.837, "dur": 2.547,
    "args": {
      "External id": 2478622,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7471872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6325
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656023025.270, "dur": 0.353,
    "args": {
      "External id": 2478623,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7996160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6326
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656023026.641, "dur": 0.777,
    "args": {
      "External id": 2478624,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8520448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6327
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656023030.124, "dur": 0.353,
    "args": {
      "External id": 2478625,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "8520704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6328
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656023031.183, "dur": 0.351,
    "args": {
      "External id": 2478626,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "9962496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6329
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656023032.039, "dur": 1.843,
    "args": {
      "External id": 2478627,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11404288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6330
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656023052.996, "dur": 28.665,
    "args": {
      "External id": 2478628,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6331
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656023145.640, "dur": 350.104,
    "args": {
      "External id": 2478629,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 6332
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514656023175.072, "dur": 315.873,
    "args": {
      "External id": 2478630,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 6333, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1336754,
    "ts": 1514656023186.376, "dur": 299.394,
    "args": {
      "External id": 2478631,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 6334
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514656023517.155, "dur": 2.160,
    "args": {
      "External id": 2478632,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 6335, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.4)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656023622.723, "dur": 16690.288,
    "args": {
      "External id": 2478633,"Record function id": 0, "Ev Idx": 6336
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656023717.154, "dur": 6.062,
    "args": {
      "External id": 2478634,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 6337
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656023726.506, "dur": 1.035,
    "args": {
      "External id": 2478635,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6338
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656023729.151, "dur": 2.764,
    "args": {
      "External id": 2478636,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6339
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656023733.886, "dur": 0.738,
    "args": {
      "External id": 2478637,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6340
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656023736.044, "dur": 0.934,
    "args": {
      "External id": 2478638,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6341
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656023738.265, "dur": 0.885,
    "args": {
      "External id": 2478639,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6342
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656023742.456, "dur": 1.275,
    "args": {
      "External id": 2478640,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6343
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656023746.514, "dur": 3.555,
    "args": {
      "External id": 2478641,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6344
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656023751.530, "dur": 0.814,
    "args": {
      "External id": 2478642,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6345
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656023753.855, "dur": 0.773,
    "args": {
      "External id": 2478643,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6346
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656023772.502, "dur": 16503.696,
    "args": {
      "External id": 2478644,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6347
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656023787.671, "dur": 16481.480,
    "args": {
      "External id": 2478645,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6348
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656023807.082, "dur": 15.228,
    "args": {
      "External id": 2478646,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6349
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514656023825.560, "dur": 16411.178,
    "args": {
      "External id": 2478647,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 6350
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656023827.872, "dur": 16408.344,
    "args": {
      "External id": 2478648,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 6351
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656023833.510, "dur": 4.661,
    "args": {
      "External id": 2478649,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6352
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656023839.826, "dur": 16393.169,
    "args": {
      "External id": 2478650,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 6353
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656040443.278, "dur": 33.252,
    "args": {
      "External id": 2478651,"Sequence number": 24740429, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 6354
    }
  },
  {
    "ph": "s", "id": 219, "pid": 1336754, "tid": 1336754, "ts": 1514656040443.278,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336754, "tid": 1336754,
    "ts": 1514656040465.143, "dur": 6.882,
    "args": {
      "External id": 2478652,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 6355
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656040468.105, "dur": 3.712,
    "args": {
      "External id": 2478653,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 6356
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336754, "tid": 1336754,
    "ts": 1514656040557.019, "dur": 75.448,
    "args": {
      "External id": 2478654,"Record function id": 0, "Ev Idx": 6357
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656040634.280, "dur": 1075.373,
    "args": {
      "External id": 2478655,"Record function id": 0, "Ev Idx": 6358
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656040673.884, "dur": 1023.533,
    "args": {
      "External id": 2478656,"Sequence number": 24740430, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 6359
    }
  },
  {
    "ph": "s", "id": 218, "pid": 1336754, "tid": 1336754, "ts": 1514656040673.884,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514656040737.796, "dur": 42.452,
    "args": {
      "External id": 2478657,"kernel_hash": "caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/ao/caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6360
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656040792.900, "dur": 102.669,
    "args": {
      "External id": 2478658,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6361
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656040906.242, "dur": 40.289,
    "args": {
      "External id": 2478659,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6362
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656040956.226, "dur": 73.296,
    "args": {
      "External id": 2478660,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6363
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656041058.221, "dur": 28.788,
    "args": {
      "External id": 2478661,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6364
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656041103.190, "dur": 14.131,
    "args": {
      "External id": 2478662,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6365
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336754, "tid": 1336754,
    "ts": 1514656041135.409, "dur": 126.805,
    "args": {
      "External id": 2478663,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 6366
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656041184.590, "dur": 11.227,
    "args": {
      "External id": 2478664,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 6367
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656041189.270, "dur": 5.770,
    "args": {
      "External id": 2478665,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6368
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656041198.834, "dur": 6.390,
    "args": {
      "External id": 2478666,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6369
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656041206.534, "dur": 0.903,
    "args": {
      "External id": 2478667,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6370
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656041209.818, "dur": 4.519,
    "args": {
      "External id": 2478668,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6371
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656041272.261, "dur": 50.440,
    "args": {
      "External id": 2478669,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6372
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336754, "tid": 1336754,
    "ts": 1514656041352.877, "dur": 28.379,
    "args": {
      "External id": 2478670,"kernel_hash": "cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/wx/cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6373
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656041389.562, "dur": 41.659,
    "args": {
      "External id": 2478671,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6374
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656041439.435, "dur": 35.204,
    "args": {
      "External id": 2478672,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6375
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514656041495.500, "dur": 28.350,
    "args": {
      "External id": 2478673,"kernel_hash": "c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2f/c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 6376
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656041550.617, "dur": 40.166,
    "args": {
      "External id": 2478674,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 6377
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656041613.278, "dur": 18.675,
    "args": {
      "External id": 2478675,"kernel_hash": "cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/gi/cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 6378
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.4)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656041769.162, "dur": 71.082,
    "args": {
      "External id": 2478676,"Record function id": 0, "Ev Idx": 6379
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656041907.412, "dur": 43.449,
    "args": {
      "External id": 2478677,"Record function id": 0, "Ev Idx": 6380
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.5)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656041959.513, "dur": 18342.910,
    "args": {
      "External id": 2478678,"Record function id": 0, "Ev Idx": 6381
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.5)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656041967.131, "dur": 929.256,
    "args": {
      "External id": 2478679,"Record function id": 0, "Ev Idx": 6382
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656042088.017, "dur": 8.709,
    "args": {
      "External id": 2478680,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6383
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514656042110.835, "dur": 38.852,
    "args": {
      "External id": 2478681,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6384
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656042116.295, "dur": 2.345,
    "args": {
      "External id": 2478682,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6385
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656042123.175, "dur": 0.500,
    "args": {
      "External id": 2478683,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6386
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656042125.256, "dur": 0.349,
    "args": {
      "External id": 2478684,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6387
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656042127.032, "dur": 0.444,
    "args": {
      "External id": 2478685,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6388
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656042130.399, "dur": 0.762,
    "args": {
      "External id": 2478686,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6389
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656042132.739, "dur": 0.376,
    "args": {
      "External id": 2478687,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6390
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656042134.735, "dur": 3.437,
    "args": {
      "External id": 2478688,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6391
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656042139.366, "dur": 0.491,
    "args": {
      "External id": 2478689,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6392
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656042141.108, "dur": 0.314,
    "args": {
      "External id": 2478690,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6393
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656042161.222, "dur": 43.970,
    "args": {
      "External id": 2478691,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6394
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1336754,
    "ts": 1514656042238.962, "dur": 115.216,
    "args": {
      "External id": 2478692,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "1", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 6395
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656042251.049, "dur": 3.562,
    "args": {
      "External id": 2478693,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6396
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1336754,
    "ts": 1514656042259.541, "dur": 9.887,
    "args": {
      "External id": 2478694,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6397
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656042263.973, "dur": 5.059,
    "args": {
      "External id": 2478695,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "12846080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 6398
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656042267.171, "dur": 0.490,
    "args": {
      "External id": 2478696,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6399
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514656042276.552, "dur": 31.583,
    "args": {
      "External id": 2478697,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6400
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656042278.903, "dur": 1.927,
    "args": {
      "External id": 2478698,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6401
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656042282.507, "dur": 0.487,
    "args": {
      "External id": 2478699,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6423296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6402
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656042284.921, "dur": 0.322,
    "args": {
      "External id": 2478700,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6947584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6403
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656042288.897, "dur": 1.832,
    "args": {
      "External id": 2478701,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7471872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6404
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656042292.390, "dur": 0.391,
    "args": {
      "External id": 2478702,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7996160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6405
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656042294.302, "dur": 0.566,
    "args": {
      "External id": 2478703,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8520448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6406
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656042297.920, "dur": 0.300,
    "args": {
      "External id": 2478704,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "8520704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6407
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656042299.689, "dur": 0.562,
    "args": {
      "External id": 2478705,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "9962496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6408
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656042301.514, "dur": 1.806,
    "args": {
      "External id": 2478706,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11404288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6409
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656042320.763, "dur": 24.933,
    "args": {
      "External id": 2478707,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6410
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656042405.286, "dur": 395.632,
    "args": {
      "External id": 2478708,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 6411
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514656042434.917, "dur": 360.800,
    "args": {
      "External id": 2478709,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 6412, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1336754,
    "ts": 1514656042446.116, "dur": 343.196,
    "args": {
      "External id": 2478710,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 6413
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514656042825.721, "dur": 2.166,
    "args": {
      "External id": 2478711,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 6414, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.5)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656042916.604, "dur": 17198.267,
    "args": {
      "External id": 2478712,"Record function id": 0, "Ev Idx": 6415
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656043054.466, "dur": 6.523,
    "args": {
      "External id": 2478713,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 6416
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656043065.795, "dur": 1.152,
    "args": {
      "External id": 2478714,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6417
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656043069.267, "dur": 3.057,
    "args": {
      "External id": 2478715,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6418
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656043074.410, "dur": 1.087,
    "args": {
      "External id": 2478716,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6419
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656043076.888, "dur": 0.918,
    "args": {
      "External id": 2478717,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6420
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656043079.524, "dur": 0.845,
    "args": {
      "External id": 2478718,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6421
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656043083.808, "dur": 0.861,
    "args": {
      "External id": 2478719,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6422
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656043086.087, "dur": 2.453,
    "args": {
      "External id": 2478720,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6423
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656043090.093, "dur": 1.164,
    "args": {
      "External id": 2478721,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6424
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656043092.848, "dur": 0.916,
    "args": {
      "External id": 2478722,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6425
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656043113.695, "dur": 16957.702,
    "args": {
      "External id": 2478723,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6426
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656043135.511, "dur": 16929.187,
    "args": {
      "External id": 2478724,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6427
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656043156.231, "dur": 15.122,
    "args": {
      "External id": 2478725,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6428
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514656043174.350, "dur": 16857.847,
    "args": {
      "External id": 2478726,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 6429
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656043176.839, "dur": 16854.818,
    "args": {
      "External id": 2478727,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 6430
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656043182.294, "dur": 4.721,
    "args": {
      "External id": 2478728,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6431
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656043188.590, "dur": 16840.020,
    "args": {
      "External id": 2478729,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 6432
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656060251.643, "dur": 27.193,
    "args": {
      "External id": 2478730,"Sequence number": 24740431, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 6433
    }
  },
  {
    "ph": "s", "id": 217, "pid": 1336754, "tid": 1336754, "ts": 1514656060251.643,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336754, "tid": 1336754,
    "ts": 1514656060267.114, "dur": 7.172,
    "args": {
      "External id": 2478731,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 6434
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656060270.233, "dur": 3.847,
    "args": {
      "External id": 2478732,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 6435
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336754, "tid": 1336754,
    "ts": 1514656060341.175, "dur": 75.532,
    "args": {
      "External id": 2478733,"Record function id": 0, "Ev Idx": 6436
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656060418.371, "dur": 1074.787,
    "args": {
      "External id": 2478734,"Record function id": 0, "Ev Idx": 6437
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656060458.600, "dur": 1021.566,
    "args": {
      "External id": 2478735,"Sequence number": 24740432, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 6438
    }
  },
  {
    "ph": "s", "id": 216, "pid": 1336754, "tid": 1336754, "ts": 1514656060458.600,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514656060519.619, "dur": 55.257,
    "args": {
      "External id": 2478736,"kernel_hash": "caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/ao/caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6439
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656060589.391, "dur": 105.781,
    "args": {
      "External id": 2478737,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6440
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656060707.253, "dur": 38.865,
    "args": {
      "External id": 2478738,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6441
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656060754.696, "dur": 31.648,
    "args": {
      "External id": 2478739,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6442
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656060810.674, "dur": 26.919,
    "args": {
      "External id": 2478740,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6443
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656060855.378, "dur": 14.878,
    "args": {
      "External id": 2478741,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6444
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336754, "tid": 1336754,
    "ts": 1514656060893.128, "dur": 172.811,
    "args": {
      "External id": 2478742,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 6445
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656060947.584, "dur": 10.248,
    "args": {
      "External id": 2478743,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 6446
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656060952.285, "dur": 4.778,
    "args": {
      "External id": 2478744,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6447
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656060960.276, "dur": 4.964,
    "args": {
      "External id": 2478745,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6448
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656060966.861, "dur": 1.243,
    "args": {
      "External id": 2478746,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6449
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656060970.582, "dur": 5.675,
    "args": {
      "External id": 2478747,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6450
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656061078.599, "dur": 53.008,
    "args": {
      "External id": 2478748,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6451
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336754, "tid": 1336754,
    "ts": 1514656061163.764, "dur": 31.712,
    "args": {
      "External id": 2478749,"kernel_hash": "cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/wx/cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6452
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656061204.664, "dur": 43.299,
    "args": {
      "External id": 2478750,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6453
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656061258.083, "dur": 35.690,
    "args": {
      "External id": 2478751,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6454
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514656061314.298, "dur": 25.358,
    "args": {
      "External id": 2478752,"kernel_hash": "c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2f/c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 6455
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656061345.315, "dur": 34.542,
    "args": {
      "External id": 2478753,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 6456
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656061400.133, "dur": 17.750,
    "args": {
      "External id": 2478754,"kernel_hash": "cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/gi/cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 6457
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.5)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656061570.568, "dur": 76.278,
    "args": {
      "External id": 2478755,"Record function id": 0, "Ev Idx": 6458
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656061719.308, "dur": 44.966,
    "args": {
      "External id": 2478756,"Record function id": 0, "Ev Idx": 6459
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.6)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656061774.689, "dur": 17857.929,
    "args": {
      "External id": 2478757,"Record function id": 0, "Ev Idx": 6460
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.6)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656061784.645, "dur": 909.518,
    "args": {
      "External id": 2478758,"Record function id": 0, "Ev Idx": 6461
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656061864.732, "dur": 7.840,
    "args": {
      "External id": 2478759,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6462
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514656061885.957, "dur": 39.610,
    "args": {
      "External id": 2478760,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6463
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656061891.272, "dur": 2.159,
    "args": {
      "External id": 2478761,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6464
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656061897.350, "dur": 0.440,
    "args": {
      "External id": 2478762,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6465
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656061899.703, "dur": 0.728,
    "args": {
      "External id": 2478763,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6466
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656061901.826, "dur": 0.562,
    "args": {
      "External id": 2478764,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6467
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656061905.139, "dur": 0.435,
    "args": {
      "External id": 2478765,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6468
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656061907.367, "dur": 0.637,
    "args": {
      "External id": 2478766,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6469
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656061909.394, "dur": 4.235,
    "args": {
      "External id": 2478767,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6470
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656061915.245, "dur": 0.534,
    "args": {
      "External id": 2478768,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6471
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656061917.024, "dur": 0.533,
    "args": {
      "External id": 2478769,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6472
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656061937.256, "dur": 42.610,
    "args": {
      "External id": 2478770,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6473
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1336754,
    "ts": 1514656062051.761, "dur": 118.501,
    "args": {
      "External id": 2478771,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "1", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 6474
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656062064.904, "dur": 5.712,
    "args": {
      "External id": 2478772,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6475
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1336754,
    "ts": 1514656062075.852, "dur": 11.010,
    "args": {
      "External id": 2478773,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6476
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656062080.167, "dur": 6.309,
    "args": {
      "External id": 2478774,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "12846080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 6477
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656062083.782, "dur": 1.173,
    "args": {
      "External id": 2478775,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6478
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514656062093.883, "dur": 30.803,
    "args": {
      "External id": 2478776,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6479
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656062096.240, "dur": 2.123,
    "args": {
      "External id": 2478777,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6480
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656062099.886, "dur": 0.416,
    "args": {
      "External id": 2478778,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6423296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6481
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656062101.640, "dur": 0.583,
    "args": {
      "External id": 2478779,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6947584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6482
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656062105.621, "dur": 2.754,
    "args": {
      "External id": 2478780,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7471872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6483
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656062109.894, "dur": 0.251,
    "args": {
      "External id": 2478781,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7996160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6484
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656062111.791, "dur": 0.366,
    "args": {
      "External id": 2478782,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8520448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6485
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656062113.491, "dur": 0.418,
    "args": {
      "External id": 2478783,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "8520704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6486
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656062115.902, "dur": 0.482,
    "args": {
      "External id": 2478784,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "9962496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6487
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656062117.646, "dur": 2.022,
    "args": {
      "External id": 2478785,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11404288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6488
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656062135.362, "dur": 26.986,
    "args": {
      "External id": 2478786,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6489
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656062223.776, "dur": 373.781,
    "args": {
      "External id": 2478787,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 6490
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514656062254.259, "dur": 337.461,
    "args": {
      "External id": 2478788,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 6491, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1336754,
    "ts": 1514656062264.787, "dur": 320.730,
    "args": {
      "External id": 2478789,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 6492
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514656062623.396, "dur": 2.898,
    "args": {
      "External id": 2478790,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 6493, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.6)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656062714.547, "dur": 16716.706,
    "args": {
      "External id": 2478791,"Record function id": 0, "Ev Idx": 6494
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656062808.947, "dur": 6.132,
    "args": {
      "External id": 2478792,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 6495
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656062819.113, "dur": 1.449,
    "args": {
      "External id": 2478793,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6496
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656062822.468, "dur": 3.194,
    "args": {
      "External id": 2478794,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6497
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656062827.462, "dur": 1.074,
    "args": {
      "External id": 2478795,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6498
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656062830.033, "dur": 1.000,
    "args": {
      "External id": 2478796,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6499
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656062832.514, "dur": 0.685,
    "args": {
      "External id": 2478797,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6500
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656062836.448, "dur": 1.084,
    "args": {
      "External id": 2478798,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6501
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656062839.016, "dur": 2.647,
    "args": {
      "External id": 2478799,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6502
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656062843.536, "dur": 1.133,
    "args": {
      "External id": 2478800,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6503
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656062846.219, "dur": 1.056,
    "args": {
      "External id": 2478801,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6504
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656062865.648, "dur": 16520.730,
    "args": {
      "External id": 2478802,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6505
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656062880.911, "dur": 16497.860,
    "args": {
      "External id": 2478803,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6506
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656062901.119, "dur": 14.836,
    "args": {
      "External id": 2478804,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6507
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514656062918.713, "dur": 16428.223,
    "args": {
      "External id": 2478805,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 6508
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656062921.163, "dur": 16425.217,
    "args": {
      "External id": 2478806,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 6509
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656062926.674, "dur": 4.883,
    "args": {
      "External id": 2478807,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6510
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656062933.232, "dur": 16410.090,
    "args": {
      "External id": 2478808,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 6511
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656079579.547, "dur": 27.832,
    "args": {
      "External id": 2478809,"Sequence number": 24740433, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 6512
    }
  },
  {
    "ph": "s", "id": 215, "pid": 1336754, "tid": 1336754, "ts": 1514656079579.547,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336754, "tid": 1336754,
    "ts": 1514656079595.061, "dur": 7.785,
    "args": {
      "External id": 2478810,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 6513
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656079598.176, "dur": 4.298,
    "args": {
      "External id": 2478811,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 6514
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336754, "tid": 1336754,
    "ts": 1514656079671.325, "dur": 76.573,
    "args": {
      "External id": 2478812,"Record function id": 0, "Ev Idx": 6515
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656079749.402, "dur": 1057.811,
    "args": {
      "External id": 2478813,"Record function id": 0, "Ev Idx": 6516
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656079789.774, "dur": 1003.857,
    "args": {
      "External id": 2478814,"Sequence number": 24740434, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 6517
    }
  },
  {
    "ph": "s", "id": 214, "pid": 1336754, "tid": 1336754, "ts": 1514656079789.774,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514656079852.431, "dur": 42.678,
    "args": {
      "External id": 2478815,"kernel_hash": "caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/ao/caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6518
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656079907.248, "dur": 127.334,
    "args": {
      "External id": 2478816,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6519
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656080048.869, "dur": 44.096,
    "args": {
      "External id": 2478817,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6520
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656080102.885, "dur": 32.431,
    "args": {
      "External id": 2478818,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6521
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656080160.914, "dur": 26.339,
    "args": {
      "External id": 2478819,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6522
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656080204.987, "dur": 14.505,
    "args": {
      "External id": 2478820,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6523
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336754, "tid": 1336754,
    "ts": 1514656080238.314, "dur": 126.311,
    "args": {
      "External id": 2478821,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 6524
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656080287.703, "dur": 12.116,
    "args": {
      "External id": 2478822,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 6525
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656080292.403, "dur": 6.518,
    "args": {
      "External id": 2478823,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6526
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656080302.711, "dur": 5.249,
    "args": {
      "External id": 2478824,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6527
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656080309.375, "dur": 0.960,
    "args": {
      "External id": 2478825,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6528
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656080312.756, "dur": 4.087,
    "args": {
      "External id": 2478826,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6529
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656080374.596, "dur": 47.225,
    "args": {
      "External id": 2478827,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6530
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336754, "tid": 1336754,
    "ts": 1514656080453.377, "dur": 30.136,
    "args": {
      "External id": 2478828,"kernel_hash": "cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/wx/cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6531
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656080493.597, "dur": 56.128,
    "args": {
      "External id": 2478829,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6532
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656080560.415, "dur": 38.958,
    "args": {
      "External id": 2478830,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6533
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514656080623.092, "dur": 26.301,
    "args": {
      "External id": 2478831,"kernel_hash": "c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2f/c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 6534
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656080655.157, "dur": 34.672,
    "args": {
      "External id": 2478832,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 6535
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656080709.097, "dur": 17.091,
    "args": {
      "External id": 2478833,"kernel_hash": "cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/gi/cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 6536
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.6)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656080868.478, "dur": 74.966,
    "args": {
      "External id": 2478834,"Record function id": 0, "Ev Idx": 6537
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656081055.868, "dur": 49.350,
    "args": {
      "External id": 2478835,"Record function id": 0, "Ev Idx": 6538
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.7)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656081115.947, "dur": 17747.857,
    "args": {
      "External id": 2478836,"Record function id": 0, "Ev Idx": 6539
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.7)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656081125.147, "dur": 922.687,
    "args": {
      "External id": 2478837,"Record function id": 0, "Ev Idx": 6540
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656081207.360, "dur": 8.430,
    "args": {
      "External id": 2478838,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6541
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514656081229.479, "dur": 40.069,
    "args": {
      "External id": 2478839,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6542
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656081234.914, "dur": 2.277,
    "args": {
      "External id": 2478840,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6543
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656081241.655, "dur": 0.461,
    "args": {
      "External id": 2478841,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6544
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656081243.692, "dur": 0.585,
    "args": {
      "External id": 2478842,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6545
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656081245.570, "dur": 0.551,
    "args": {
      "External id": 2478843,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6546
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656081249.126, "dur": 0.856,
    "args": {
      "External id": 2478844,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6547
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656081251.649, "dur": 0.520,
    "args": {
      "External id": 2478845,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6548
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656081253.599, "dur": 4.214,
    "args": {
      "External id": 2478846,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6549
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656081259.562, "dur": 0.486,
    "args": {
      "External id": 2478847,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6550
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656081261.588, "dur": 0.692,
    "args": {
      "External id": 2478848,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6551
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656081281.045, "dur": 41.361,
    "args": {
      "External id": 2478849,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6552
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1336754,
    "ts": 1514656081355.110, "dur": 119.804,
    "args": {
      "External id": 2478850,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "1", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 6553
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656081367.347, "dur": 3.169,
    "args": {
      "External id": 2478851,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6554
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1336754,
    "ts": 1514656081375.628, "dur": 10.061,
    "args": {
      "External id": 2478852,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6555
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656081379.965, "dur": 5.355,
    "args": {
      "External id": 2478853,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "12846080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 6556
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656081383.469, "dur": 0.525,
    "args": {
      "External id": 2478854,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6557
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514656081392.625, "dur": 32.792,
    "args": {
      "External id": 2478855,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6558
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656081394.729, "dur": 2.194,
    "args": {
      "External id": 2478856,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6559
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656081398.444, "dur": 0.407,
    "args": {
      "External id": 2478857,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6423296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6560
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656081400.555, "dur": 0.513,
    "args": {
      "External id": 2478858,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6947584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6561
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656081404.185, "dur": 2.157,
    "args": {
      "External id": 2478859,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7471872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6562
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656081408.059, "dur": 0.746,
    "args": {
      "External id": 2478860,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7996160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6563
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656081410.061, "dur": 0.398,
    "args": {
      "External id": 2478861,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8520448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6564
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656081413.245, "dur": 0.335,
    "args": {
      "External id": 2478862,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "8520704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6565
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656081415.441, "dur": 0.379,
    "args": {
      "External id": 2478863,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "9962496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6566
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656081417.280, "dur": 2.241,
    "args": {
      "External id": 2478864,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11404288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6567
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656081437.457, "dur": 29.088,
    "args": {
      "External id": 2478865,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6568
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656081524.971, "dur": 394.433,
    "args": {
      "External id": 2478866,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 6569
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514656081571.175, "dur": 342.769,
    "args": {
      "External id": 2478867,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 6570, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1336754,
    "ts": 1514656081582.972, "dur": 324.733,
    "args": {
      "External id": 2478868,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 6571
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514656081942.533, "dur": 2.269,
    "args": {
      "External id": 2478869,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 6572, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.7)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656082070.139, "dur": 16607.632,
    "args": {
      "External id": 2478870,"Record function id": 0, "Ev Idx": 6573
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656082179.346, "dur": 6.573,
    "args": {
      "External id": 2478871,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 6574
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656082191.362, "dur": 1.062,
    "args": {
      "External id": 2478872,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6575
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656082194.374, "dur": 3.417,
    "args": {
      "External id": 2478873,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6576
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656082201.634, "dur": 0.981,
    "args": {
      "External id": 2478874,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6577
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656082204.300, "dur": 1.255,
    "args": {
      "External id": 2478875,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6578
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656082207.212, "dur": 1.202,
    "args": {
      "External id": 2478876,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6579
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656082210.238, "dur": 1.144,
    "args": {
      "External id": 2478877,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6580
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656082214.568, "dur": 3.175,
    "args": {
      "External id": 2478878,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6581
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656082219.289, "dur": 1.065,
    "args": {
      "External id": 2478879,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6582
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656082221.975, "dur": 0.715,
    "args": {
      "External id": 2478880,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6583
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656082241.747, "dur": 16391.089,
    "args": {
      "External id": 2478881,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6584
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656082257.118, "dur": 16368.422,
    "args": {
      "External id": 2478882,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6585
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656082275.840, "dur": 15.242,
    "args": {
      "External id": 2478883,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6586
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514656082296.057, "dur": 16296.859,
    "args": {
      "External id": 2478884,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 6587
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656082298.469, "dur": 16293.926,
    "args": {
      "External id": 2478885,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 6588
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656082303.797, "dur": 4.908,
    "args": {
      "External id": 2478886,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6589
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656082310.262, "dur": 16279.409,
    "args": {
      "External id": 2478887,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 6590
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656098811.499, "dur": 28.608,
    "args": {
      "External id": 2478888,"Sequence number": 24740435, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 6591
    }
  },
  {
    "ph": "s", "id": 213, "pid": 1336754, "tid": 1336754, "ts": 1514656098811.499,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336754, "tid": 1336754,
    "ts": 1514656098828.227, "dur": 7.120,
    "args": {
      "External id": 2478889,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 6592
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656098831.262, "dur": 3.867,
    "args": {
      "External id": 2478890,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 6593
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336754, "tid": 1336754,
    "ts": 1514656098900.428, "dur": 73.311,
    "args": {
      "External id": 2478891,"Record function id": 0, "Ev Idx": 6594
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656098975.495, "dur": 1096.010,
    "args": {
      "External id": 2478892,"Record function id": 0, "Ev Idx": 6595
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656099046.806, "dur": 1011.130,
    "args": {
      "External id": 2478893,"Sequence number": 24740436, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 6596
    }
  },
  {
    "ph": "s", "id": 212, "pid": 1336754, "tid": 1336754, "ts": 1514656099046.806,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514656099113.332, "dur": 44.218,
    "args": {
      "External id": 2478894,"kernel_hash": "caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/ao/caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6597
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656099169.945, "dur": 104.041,
    "args": {
      "External id": 2478895,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6598
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656099284.128, "dur": 39.836,
    "args": {
      "External id": 2478896,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6599
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656099332.679, "dur": 31.605,
    "args": {
      "External id": 2478897,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6600
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656099388.097, "dur": 23.007,
    "args": {
      "External id": 2478898,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6601
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656099427.955, "dur": 14.807,
    "args": {
      "External id": 2478899,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6602
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336754, "tid": 1336754,
    "ts": 1514656099460.082, "dur": 154.539,
    "args": {
      "External id": 2478900,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 6603
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656099510.126, "dur": 10.134,
    "args": {
      "External id": 2478901,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 6604
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656099514.365, "dur": 5.148,
    "args": {
      "External id": 2478902,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6605
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656099522.865, "dur": 23.620,
    "args": {
      "External id": 2478903,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6606
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656099554.337, "dur": 1.423,
    "args": {
      "External id": 2478904,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6607
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656099558.271, "dur": 4.487,
    "args": {
      "External id": 2478905,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6608
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656099625.841, "dur": 50.070,
    "args": {
      "External id": 2478906,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6609
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336754, "tid": 1336754,
    "ts": 1514656099708.503, "dur": 27.529,
    "args": {
      "External id": 2478907,"kernel_hash": "cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/wx/cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6610
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656099745.692, "dur": 43.576,
    "args": {
      "External id": 2478908,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6611
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656099797.314, "dur": 36.144,
    "args": {
      "External id": 2478909,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6612
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514656099854.696, "dur": 26.494,
    "args": {
      "External id": 2478910,"kernel_hash": "c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2f/c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 6613
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656099886.718, "dur": 33.395,
    "args": {
      "External id": 2478911,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 6614
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656099940.311, "dur": 16.051,
    "args": {
      "External id": 2478912,"kernel_hash": "cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/gi/cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 6615
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.7)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656100134.629, "dur": 75.792,
    "args": {
      "External id": 2478913,"Record function id": 0, "Ev Idx": 6616
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656100281.128, "dur": 47.095,
    "args": {
      "External id": 2478914,"Record function id": 0, "Ev Idx": 6617
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.8)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656100337.126, "dur": 17998.464,
    "args": {
      "External id": 2478915,"Record function id": 0, "Ev Idx": 6618
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.8)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656100345.082, "dur": 930.359,
    "args": {
      "External id": 2478916,"Record function id": 0, "Ev Idx": 6619
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656100425.483, "dur": 7.946,
    "args": {
      "External id": 2478917,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6620
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514656100445.977, "dur": 38.360,
    "args": {
      "External id": 2478918,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6621
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656100451.498, "dur": 2.095,
    "args": {
      "External id": 2478919,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6622
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656100458.080, "dur": 0.462,
    "args": {
      "External id": 2478920,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6623
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656100459.810, "dur": 0.793,
    "args": {
      "External id": 2478921,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6624
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656100462.129, "dur": 0.744,
    "args": {
      "External id": 2478922,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6625
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656100465.321, "dur": 0.605,
    "args": {
      "External id": 2478923,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6626
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656100467.166, "dur": 0.442,
    "args": {
      "External id": 2478924,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6627
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656100468.646, "dur": 4.092,
    "args": {
      "External id": 2478925,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6628
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656100474.535, "dur": 0.244,
    "args": {
      "External id": 2478926,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6629
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656100475.937, "dur": 0.250,
    "args": {
      "External id": 2478927,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6630
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656100495.497, "dur": 58.506,
    "args": {
      "External id": 2478928,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6631
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1336754,
    "ts": 1514656100590.864, "dur": 120.256,
    "args": {
      "External id": 2478929,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "1", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 6632
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656100603.473, "dur": 4.489,
    "args": {
      "External id": 2478930,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6633
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1336754,
    "ts": 1514656100612.627, "dur": 10.009,
    "args": {
      "External id": 2478931,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6634
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656100616.710, "dur": 5.545,
    "args": {
      "External id": 2478932,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "12846080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 6635
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656100619.917, "dur": 0.881,
    "args": {
      "External id": 2478933,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6636
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514656100629.534, "dur": 36.718,
    "args": {
      "External id": 2478934,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6637
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656100631.277, "dur": 2.449,
    "args": {
      "External id": 2478935,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6638
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656100635.796, "dur": 0.398,
    "args": {
      "External id": 2478936,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6423296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6639
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656100637.530, "dur": 0.462,
    "args": {
      "External id": 2478937,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6947584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6640
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656100641.489, "dur": 2.084,
    "args": {
      "External id": 2478938,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7471872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6641
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656100644.682, "dur": 0.605,
    "args": {
      "External id": 2478939,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7996160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6642
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656100646.829, "dur": 0.524,
    "args": {
      "External id": 2478940,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8520448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6643
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656100648.596, "dur": 0.373,
    "args": {
      "External id": 2478941,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "8520704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6644
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656100650.325, "dur": 6.351,
    "args": {
      "External id": 2478942,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "9962496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6645
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656100657.855, "dur": 1.977,
    "args": {
      "External id": 2478943,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11404288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6646
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656100677.363, "dur": 25.534,
    "args": {
      "External id": 2478944,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6647
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656100765.182, "dur": 409.810,
    "args": {
      "External id": 2478945,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 6648
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514656100795.458, "dur": 374.082,
    "args": {
      "External id": 2478946,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 6649, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1336754,
    "ts": 1514656100806.192, "dur": 357.136,
    "args": {
      "External id": 2478947,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 6650
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514656101201.467, "dur": 2.459,
    "args": {
      "External id": 2478948,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 6651, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.8)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656101296.353, "dur": 16852.328,
    "args": {
      "External id": 2478949,"Record function id": 0, "Ev Idx": 6652
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656101395.644, "dur": 6.660,
    "args": {
      "External id": 2478950,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 6653
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656101406.570, "dur": 1.403,
    "args": {
      "External id": 2478951,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6654
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656101410.104, "dur": 3.077,
    "args": {
      "External id": 2478952,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6655
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656101414.948, "dur": 1.491,
    "args": {
      "External id": 2478953,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6656
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656101417.799, "dur": 1.201,
    "args": {
      "External id": 2478954,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6657
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656101420.460, "dur": 0.919,
    "args": {
      "External id": 2478955,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6658
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656101424.543, "dur": 1.254,
    "args": {
      "External id": 2478956,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6659
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656101427.215, "dur": 2.712,
    "args": {
      "External id": 2478957,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6660
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656101431.651, "dur": 0.896,
    "args": {
      "External id": 2478958,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6661
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656101433.995, "dur": 0.759,
    "args": {
      "External id": 2478959,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6662
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656101463.192, "dur": 16637.408,
    "args": {
      "External id": 2478960,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6663
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656101479.479, "dur": 16613.894,
    "args": {
      "External id": 2478961,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6664
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656101497.520, "dur": 13.912,
    "args": {
      "External id": 2478962,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6665
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514656101514.367, "dur": 16546.748,
    "args": {
      "External id": 2478963,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 6666
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656101517.221, "dur": 16543.268,
    "args": {
      "External id": 2478964,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 6667
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656101522.820, "dur": 22.930,
    "args": {
      "External id": 2478965,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6668
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656101548.183, "dur": 16509.253,
    "args": {
      "External id": 2478966,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 6669
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656118284.054, "dur": 27.356,
    "args": {
      "External id": 2478967,"Sequence number": 24740437, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 6670
    }
  },
  {
    "ph": "s", "id": 211, "pid": 1336754, "tid": 1336754, "ts": 1514656118284.054,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336754, "tid": 1336754,
    "ts": 1514656118299.787, "dur": 6.926,
    "args": {
      "External id": 2478968,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 6671
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656118302.344, "dur": 4.116,
    "args": {
      "External id": 2478969,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 6672
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336754, "tid": 1336754,
    "ts": 1514656118372.865, "dur": 76.099,
    "args": {
      "External id": 2478970,"Record function id": 0, "Ev Idx": 6673
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656118450.506, "dur": 1055.001,
    "args": {
      "External id": 2478971,"Record function id": 0, "Ev Idx": 6674
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656118488.008, "dur": 1005.682,
    "args": {
      "External id": 2478972,"Sequence number": 24740438, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 6675
    }
  },
  {
    "ph": "s", "id": 210, "pid": 1336754, "tid": 1336754, "ts": 1514656118488.008,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514656118564.063, "dur": 41.926,
    "args": {
      "External id": 2478973,"kernel_hash": "caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/ao/caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6676
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656118620.091, "dur": 103.464,
    "args": {
      "External id": 2478974,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6677
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656118734.621, "dur": 39.260,
    "args": {
      "External id": 2478975,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6678
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656118783.340, "dur": 31.624,
    "args": {
      "External id": 2478976,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6679
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656118838.708, "dur": 25.543,
    "args": {
      "External id": 2478977,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6680
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656118881.157, "dur": 14.848,
    "args": {
      "External id": 2478978,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6681
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336754, "tid": 1336754,
    "ts": 1514656118915.270, "dur": 172.032,
    "args": {
      "External id": 2478979,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 6682
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656118964.330, "dur": 9.998,
    "args": {
      "External id": 2478980,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 6683
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656118968.605, "dur": 4.895,
    "args": {
      "External id": 2478981,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6684
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656118977.176, "dur": 43.419,
    "args": {
      "External id": 2478982,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6685
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656119023.354, "dur": 1.436,
    "args": {
      "External id": 2478983,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6686
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656119027.464, "dur": 4.990,
    "args": {
      "External id": 2478984,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6687
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656119099.272, "dur": 50.981,
    "args": {
      "External id": 2478985,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6688
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336754, "tid": 1336754,
    "ts": 1514656119182.094, "dur": 27.621,
    "args": {
      "External id": 2478986,"kernel_hash": "cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/wx/cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6689
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656119218.878, "dur": 42.285,
    "args": {
      "External id": 2478987,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6690
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656119269.570, "dur": 34.597,
    "args": {
      "External id": 2478988,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6691
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514656119326.737, "dur": 26.052,
    "args": {
      "External id": 2478989,"kernel_hash": "c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2f/c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 6692
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656119358.671, "dur": 34.782,
    "args": {
      "External id": 2478990,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 6693
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656119411.893, "dur": 17.503,
    "args": {
      "External id": 2478991,"kernel_hash": "cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/gi/cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 6694
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.8)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656119584.320, "dur": 75.275,
    "args": {
      "External id": 2478992,"Record function id": 0, "Ev Idx": 6695
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656119730.301, "dur": 44.549,
    "args": {
      "External id": 2478993,"Record function id": 0, "Ev Idx": 6696
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.9)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656119784.753, "dur": 18067.246,
    "args": {
      "External id": 2478994,"Record function id": 0, "Ev Idx": 6697
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.9)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656119793.193, "dur": 969.850,
    "args": {
      "External id": 2478995,"Record function id": 0, "Ev Idx": 6698
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656119872.193, "dur": 7.935,
    "args": {
      "External id": 2478996,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6699
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514656119893.102, "dur": 34.572,
    "args": {
      "External id": 2478997,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6700
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656119898.179, "dur": 2.130,
    "args": {
      "External id": 2478998,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6701
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656119903.686, "dur": 0.368,
    "args": {
      "External id": 2478999,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6702
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656119905.292, "dur": 0.372,
    "args": {
      "External id": 2479000,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6703
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656119906.767, "dur": 0.561,
    "args": {
      "External id": 2479001,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6704
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656119910.118, "dur": 0.570,
    "args": {
      "External id": 2479002,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6705
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656119911.526, "dur": 0.522,
    "args": {
      "External id": 2479003,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6706
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656119913.208, "dur": 3.493,
    "args": {
      "External id": 2479004,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6707
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656119917.729, "dur": 0.331,
    "args": {
      "External id": 2479005,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6708
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656119919.385, "dur": 0.653,
    "args": {
      "External id": 2479006,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6709
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656119938.838, "dur": 79.644,
    "args": {
      "External id": 2479007,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6710
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1336754,
    "ts": 1514656120056.186, "dur": 126.522,
    "args": {
      "External id": 2479008,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "1", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 6711
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656120068.971, "dur": 5.246,
    "args": {
      "External id": 2479009,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6712
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1336754,
    "ts": 1514656120079.415, "dur": 10.655,
    "args": {
      "External id": 2479010,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6713
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656120083.820, "dur": 5.870,
    "args": {
      "External id": 2479011,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "12846080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 6714
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656120086.982, "dur": 1.015,
    "args": {
      "External id": 2479012,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6715
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514656120097.126, "dur": 31.808,
    "args": {
      "External id": 2479013,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6716
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656120099.060, "dur": 2.122,
    "args": {
      "External id": 2479014,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6717
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656120105.267, "dur": 0.341,
    "args": {
      "External id": 2479015,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6423296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6718
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656120107.593, "dur": 0.542,
    "args": {
      "External id": 2479016,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6947584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6719
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656120110.817, "dur": 2.200,
    "args": {
      "External id": 2479017,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7471872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6720
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656120113.927, "dur": 0.567,
    "args": {
      "External id": 2479018,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7996160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6721
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656120115.766, "dur": 1.816,
    "args": {
      "External id": 2479019,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8520448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6722
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656120118.455, "dur": 0.526,
    "args": {
      "External id": 2479020,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "8520704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6723
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656120119.980, "dur": 0.294,
    "args": {
      "External id": 2479021,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "9962496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6724
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656120122.910, "dur": 0.465,
    "args": {
      "External id": 2479022,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11404288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6725
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656120147.250, "dur": 27.351,
    "args": {
      "External id": 2479023,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6726
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656120237.511, "dur": 427.164,
    "args": {
      "External id": 2479024,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 6727
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514656120266.852, "dur": 392.276,
    "args": {
      "External id": 2479025,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 6728, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1336754,
    "ts": 1514656120278.010, "dur": 374.912,
    "args": {
      "External id": 2479026,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 6729
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514656120689.993, "dur": 2.430,
    "args": {
      "External id": 2479027,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 6730, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.9)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656120783.189, "dur": 16885.377,
    "args": {
      "External id": 2479028,"Record function id": 0, "Ev Idx": 6731
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656120879.427, "dur": 6.217,
    "args": {
      "External id": 2479029,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 6732
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656120889.857, "dur": 1.032,
    "args": {
      "External id": 2479030,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6733
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656120892.512, "dur": 3.192,
    "args": {
      "External id": 2479031,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6734
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656120897.514, "dur": 0.925,
    "args": {
      "External id": 2479032,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6735
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656120899.650, "dur": 1.176,
    "args": {
      "External id": 2479033,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6736
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656120902.315, "dur": 1.523,
    "args": {
      "External id": 2479034,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6737
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656120906.955, "dur": 1.335,
    "args": {
      "External id": 2479035,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6738
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656120909.589, "dur": 2.152,
    "args": {
      "External id": 2479036,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6739
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656120913.261, "dur": 0.737,
    "args": {
      "External id": 2479037,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6740
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656120915.650, "dur": 1.019,
    "args": {
      "External id": 2479038,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6741
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656120934.970, "dur": 16687.773,
    "args": {
      "External id": 2479039,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6742
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656120949.910, "dur": 16666.064,
    "args": {
      "External id": 2479040,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6743
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656120969.313, "dur": 49.398,
    "args": {
      "External id": 2479041,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6744
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514656121022.611, "dur": 16562.310,
    "args": {
      "External id": 2479042,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 6745
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656121025.085, "dur": 16559.339,
    "args": {
      "External id": 2479043,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 6746
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656121030.483, "dur": 7.193,
    "args": {
      "External id": 2479044,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6747
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656121039.253, "dur": 16542.314,
    "args": {
      "External id": 2479045,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 6748
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656137801.105, "dur": 27.494,
    "args": {
      "External id": 2479046,"Sequence number": 24740439, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 6749
    }
  },
  {
    "ph": "s", "id": 209, "pid": 1336754, "tid": 1336754, "ts": 1514656137801.105,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336754, "tid": 1336754,
    "ts": 1514656137816.868, "dur": 7.018,
    "args": {
      "External id": 2479047,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 6750
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656137819.767, "dur": 3.908,
    "args": {
      "External id": 2479048,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 6751
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336754, "tid": 1336754,
    "ts": 1514656137888.974, "dur": 77.228,
    "args": {
      "External id": 2479049,"Record function id": 0, "Ev Idx": 6752
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656137967.885, "dur": 1100.564,
    "args": {
      "External id": 2479050,"Record function id": 0, "Ev Idx": 6753
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656138037.809, "dur": 1017.257,
    "args": {
      "External id": 2479051,"Sequence number": 24740440, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 6754
    }
  },
  {
    "ph": "s", "id": 208, "pid": 1336754, "tid": 1336754, "ts": 1514656138037.809,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514656138106.472, "dur": 41.037,
    "args": {
      "External id": 2479052,"kernel_hash": "caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/ao/caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6755
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656138160.836, "dur": 102.555,
    "args": {
      "External id": 2479053,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6756
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656138274.170, "dur": 45.915,
    "args": {
      "External id": 2479054,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6757
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656138327.996, "dur": 31.511,
    "args": {
      "External id": 2479055,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6758
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656138384.530, "dur": 23.804,
    "args": {
      "External id": 2479056,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6759
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656138424.860, "dur": 14.755,
    "args": {
      "External id": 2479057,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6760
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336754, "tid": 1336754,
    "ts": 1514656138458.284, "dur": 147.906,
    "args": {
      "External id": 2479058,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 6761
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656138506.271, "dur": 9.987,
    "args": {
      "External id": 2479059,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 6762
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656138510.684, "dur": 4.838,
    "args": {
      "External id": 2479060,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6763
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656138518.623, "dur": 5.343,
    "args": {
      "External id": 2479061,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6764
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656138525.662, "dur": 17.505,
    "args": {
      "External id": 2479062,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6765
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656138546.979, "dur": 6.756,
    "args": {
      "External id": 2479063,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6766
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656138617.770, "dur": 50.715,
    "args": {
      "External id": 2479064,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6767
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336754, "tid": 1336754,
    "ts": 1514656138697.796, "dur": 28.179,
    "args": {
      "External id": 2479065,"kernel_hash": "cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/wx/cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6768
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656138741.662, "dur": 42.505,
    "args": {
      "External id": 2479066,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6769
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656138792.364, "dur": 35.047,
    "args": {
      "External id": 2479067,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6770
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514656138851.359, "dur": 26.758,
    "args": {
      "External id": 2479068,"kernel_hash": "c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2f/c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 6771
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656138883.563, "dur": 34.033,
    "args": {
      "External id": 2479069,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 6772
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656138937.310, "dur": 16.782,
    "args": {
      "External id": 2479070,"kernel_hash": "cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/gi/cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 6773
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.9)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656139130.571, "dur": 72.522,
    "args": {
      "External id": 2479071,"Record function id": 0, "Ev Idx": 6774
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656139273.312, "dur": 45.894,
    "args": {
      "External id": 2479072,"Record function id": 0, "Ev Idx": 6775
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.10)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656139327.909, "dur": 17981.334,
    "args": {
      "External id": 2479073,"Record function id": 0, "Ev Idx": 6776
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.10)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656139336.447, "dur": 947.715,
    "args": {
      "External id": 2479074,"Record function id": 0, "Ev Idx": 6777
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656139418.236, "dur": 8.067,
    "args": {
      "External id": 2479075,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6778
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514656139439.139, "dur": 34.475,
    "args": {
      "External id": 2479076,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6779
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656139443.976, "dur": 2.048,
    "args": {
      "External id": 2479077,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6780
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656139450.172, "dur": 0.408,
    "args": {
      "External id": 2479078,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6781
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656139451.549, "dur": 0.530,
    "args": {
      "External id": 2479079,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6782
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656139453.160, "dur": 0.523,
    "args": {
      "External id": 2479080,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6783
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656139456.543, "dur": 0.708,
    "args": {
      "External id": 2479081,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6784
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656139458.675, "dur": 0.772,
    "args": {
      "External id": 2479082,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6785
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656139460.466, "dur": 3.473,
    "args": {
      "External id": 2479083,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6786
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656139465.041, "dur": 0.427,
    "args": {
      "External id": 2479084,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6787
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656139467.468, "dur": 0.237,
    "args": {
      "External id": 2479085,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6788
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656139484.814, "dur": 40.219,
    "args": {
      "External id": 2479086,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6789
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1336754,
    "ts": 1514656139577.302, "dur": 117.368,
    "args": {
      "External id": 2479087,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "1", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 6790
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656139589.964, "dur": 5.209,
    "args": {
      "External id": 2479088,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6791
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1336754,
    "ts": 1514656139600.451, "dur": 10.415,
    "args": {
      "External id": 2479089,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6792
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656139604.646, "dur": 5.816,
    "args": {
      "External id": 2479090,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "12846080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 6793
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656139608.133, "dur": 0.854,
    "args": {
      "External id": 2479091,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6794
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514656139618.151, "dur": 31.497,
    "args": {
      "External id": 2479092,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6795
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656139620.478, "dur": 1.933,
    "args": {
      "External id": 2479093,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6796
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656139624.724, "dur": 0.382,
    "args": {
      "External id": 2479094,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6423296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6797
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656139626.479, "dur": 0.385,
    "args": {
      "External id": 2479095,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6947584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6798
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656139630.399, "dur": 2.427,
    "args": {
      "External id": 2479096,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7471872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6799
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656139634.227, "dur": 0.271,
    "args": {
      "External id": 2479097,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7996160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6800
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656139635.837, "dur": 0.271,
    "args": {
      "External id": 2479098,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8520448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6801
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656139639.186, "dur": 0.541,
    "args": {
      "External id": 2479099,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "8520704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6802
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656139641.312, "dur": 0.405,
    "args": {
      "External id": 2479100,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "9962496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6803
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656139642.989, "dur": 2.114,
    "args": {
      "External id": 2479101,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11404288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6804
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656139660.176, "dur": 26.361,
    "args": {
      "External id": 2479102,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6805
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656139748.633, "dur": 435.371,
    "args": {
      "External id": 2479103,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 6806
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514656139778.310, "dur": 400.105,
    "args": {
      "External id": 2479104,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 6807, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1336754,
    "ts": 1514656139812.725, "dur": 360.048,
    "args": {
      "External id": 2479105,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 6808
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514656140210.373, "dur": 2.470,
    "args": {
      "External id": 2479106,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 6809, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.10)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656140304.786, "dur": 16812.337,
    "args": {
      "External id": 2479107,"Record function id": 0, "Ev Idx": 6810
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656140401.101, "dur": 6.469,
    "args": {
      "External id": 2479108,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 6811
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656140410.869, "dur": 1.233,
    "args": {
      "External id": 2479109,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6812
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656140413.927, "dur": 3.091,
    "args": {
      "External id": 2479110,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6813
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656140418.554, "dur": 1.106,
    "args": {
      "External id": 2479111,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6814
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656140421.636, "dur": 1.235,
    "args": {
      "External id": 2479112,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6815
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656140424.217, "dur": 0.788,
    "args": {
      "External id": 2479113,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6816
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656140428.230, "dur": 1.072,
    "args": {
      "External id": 2479114,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6817
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656140430.761, "dur": 2.110,
    "args": {
      "External id": 2479115,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6818
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656140434.357, "dur": 0.662,
    "args": {
      "External id": 2479116,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6819
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656140436.568, "dur": 1.095,
    "args": {
      "External id": 2479117,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6820
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656140456.737, "dur": 16621.224,
    "args": {
      "External id": 2479118,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6821
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656140472.412, "dur": 16598.558,
    "args": {
      "External id": 2479119,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6822
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656140490.129, "dur": 15.154,
    "args": {
      "External id": 2479120,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6823
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514656140508.316, "dur": 16531.074,
    "args": {
      "External id": 2479121,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 6824
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656140510.845, "dur": 16528.000,
    "args": {
      "External id": 2479122,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 6825
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656140516.447, "dur": 5.954,
    "args": {
      "External id": 2479123,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6826
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656140523.903, "dur": 16512.071,
    "args": {
      "External id": 2479124,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 6827
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656157250.495, "dur": 33.992,
    "args": {
      "External id": 2479125,"Sequence number": 24740441, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 6828
    }
  },
  {
    "ph": "s", "id": 207, "pid": 1336754, "tid": 1336754, "ts": 1514656157250.495,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336754, "tid": 1336754,
    "ts": 1514656157272.715, "dur": 7.149,
    "args": {
      "External id": 2479126,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 6829
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656157275.706, "dur": 3.963,
    "args": {
      "External id": 2479127,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 6830
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336754, "tid": 1336754,
    "ts": 1514656157347.422, "dur": 73.352,
    "args": {
      "External id": 2479128,"Record function id": 0, "Ev Idx": 6831
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656157422.352, "dur": 1048.695,
    "args": {
      "External id": 2479129,"Record function id": 0, "Ev Idx": 6832
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656157460.824, "dur": 997.215,
    "args": {
      "External id": 2479130,"Sequence number": 24740442, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 6833
    }
  },
  {
    "ph": "s", "id": 206, "pid": 1336754, "tid": 1336754, "ts": 1514656157460.824,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514656157521.945, "dur": 56.732,
    "args": {
      "External id": 2479131,"kernel_hash": "caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/ao/caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6834
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656157595.080, "dur": 104.455,
    "args": {
      "External id": 2479132,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6835
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656157709.827, "dur": 39.200,
    "args": {
      "External id": 2479133,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6836
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656157756.895, "dur": 31.704,
    "args": {
      "External id": 2479134,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6837
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656157814.027, "dur": 23.759,
    "args": {
      "External id": 2479135,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6838
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656157854.473, "dur": 13.771,
    "args": {
      "External id": 2479136,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6839
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336754, "tid": 1336754,
    "ts": 1514656157887.291, "dur": 161.704,
    "args": {
      "External id": 2479137,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 6840
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656157936.306, "dur": 10.186,
    "args": {
      "External id": 2479138,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 6841
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656157940.652, "dur": 5.049,
    "args": {
      "External id": 2479139,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6842
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656157949.130, "dur": 5.653,
    "args": {
      "External id": 2479140,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6843
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656157956.391, "dur": 0.975,
    "args": {
      "External id": 2479141,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6844
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656157959.995, "dur": 4.948,
    "args": {
      "External id": 2479142,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6845
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656158061.114, "dur": 53.409,
    "args": {
      "External id": 2479143,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6846
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336754, "tid": 1336754,
    "ts": 1514656158145.889, "dur": 28.970,
    "args": {
      "External id": 2479144,"kernel_hash": "cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/wx/cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6847
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656158186.259, "dur": 43.501,
    "args": {
      "External id": 2479145,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6848
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656158237.721, "dur": 35.172,
    "args": {
      "External id": 2479146,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6849
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514656158294.369, "dur": 25.851,
    "args": {
      "External id": 2479147,"kernel_hash": "c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2f/c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 6850
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656158326.231, "dur": 35.217,
    "args": {
      "External id": 2479148,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 6851
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656158380.147, "dur": 16.758,
    "args": {
      "External id": 2479149,"kernel_hash": "cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/gi/cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 6852
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.10)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656158545.764, "dur": 75.867,
    "args": {
      "External id": 2479150,"Record function id": 0, "Ev Idx": 6853
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656158692.630, "dur": 46.769,
    "args": {
      "External id": 2479151,"Record function id": 0, "Ev Idx": 6854
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.11)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656158748.191, "dur": 17864.483,
    "args": {
      "External id": 2479152,"Record function id": 0, "Ev Idx": 6855
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.11)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656158757.594, "dur": 906.548,
    "args": {
      "External id": 2479153,"Record function id": 0, "Ev Idx": 6856
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656158837.728, "dur": 7.952,
    "args": {
      "External id": 2479154,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6857
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514656158858.811, "dur": 37.105,
    "args": {
      "External id": 2479155,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6858
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656158864.173, "dur": 2.312,
    "args": {
      "External id": 2479156,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6859
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656158870.807, "dur": 0.399,
    "args": {
      "External id": 2479157,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6860
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656158873.224, "dur": 0.362,
    "args": {
      "External id": 2479158,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6861
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656158875.052, "dur": 0.514,
    "args": {
      "External id": 2479159,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6862
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656158878.407, "dur": 0.714,
    "args": {
      "External id": 2479160,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6863
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656158880.363, "dur": 0.737,
    "args": {
      "External id": 2479161,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6864
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656158882.499, "dur": 3.667,
    "args": {
      "External id": 2479162,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6865
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656158887.543, "dur": 0.444,
    "args": {
      "External id": 2479163,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6866
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656158889.245, "dur": 0.548,
    "args": {
      "External id": 2479164,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6867
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656158908.771, "dur": 42.506,
    "args": {
      "External id": 2479165,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6868
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1336754,
    "ts": 1514656159017.831, "dur": 125.826,
    "args": {
      "External id": 2479166,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "1", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 6869
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656159031.518, "dur": 7.124,
    "args": {
      "External id": 2479167,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6870
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1336754,
    "ts": 1514656159044.391, "dur": 11.106,
    "args": {
      "External id": 2479168,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6871
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656159048.682, "dur": 6.433,
    "args": {
      "External id": 2479169,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "12846080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 6872
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656159052.504, "dur": 0.811,
    "args": {
      "External id": 2479170,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6873
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514656159063.298, "dur": 30.028,
    "args": {
      "External id": 2479171,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6874
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656159065.415, "dur": 0.706,
    "args": {
      "External id": 2479172,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6875
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656159067.748, "dur": 2.100,
    "args": {
      "External id": 2479173,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6423296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6876
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656159071.368, "dur": 0.411,
    "args": {
      "External id": 2479174,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6947584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6877
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656159073.273, "dur": 2.464,
    "args": {
      "External id": 2479175,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7471872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6878
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656159079.273, "dur": 0.382,
    "args": {
      "External id": 2479176,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7996160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6879
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656159080.904, "dur": 0.486,
    "args": {
      "External id": 2479177,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8520448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6880
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656159082.729, "dur": 0.560,
    "args": {
      "External id": 2479178,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "8520704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6881
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656159086.294, "dur": 0.287,
    "args": {
      "External id": 2479179,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "9962496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6882
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656159087.735, "dur": 0.504,
    "args": {
      "External id": 2479180,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11404288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6883
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656159106.532, "dur": 28.942,
    "args": {
      "External id": 2479181,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6884
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656159197.911, "dur": 371.521,
    "args": {
      "External id": 2479182,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 6885
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514656159226.444, "dur": 337.374,
    "args": {
      "External id": 2479183,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 6886, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1336754,
    "ts": 1514656159237.445, "dur": 319.347,
    "args": {
      "External id": 2479184,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 6887
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514656159593.974, "dur": 2.335,
    "args": {
      "External id": 2479185,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 6888, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.11)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656159684.862, "dur": 16720.202,
    "args": {
      "External id": 2479186,"Record function id": 0, "Ev Idx": 6889
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656159782.369, "dur": 6.381,
    "args": {
      "External id": 2479187,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 6890
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656159792.098, "dur": 1.257,
    "args": {
      "External id": 2479188,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6891
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656159795.051, "dur": 2.579,
    "args": {
      "External id": 2479189,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6892
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656159799.291, "dur": 0.848,
    "args": {
      "External id": 2479190,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6893
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656159801.614, "dur": 0.873,
    "args": {
      "External id": 2479191,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6894
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656159803.669, "dur": 0.888,
    "args": {
      "External id": 2479192,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6895
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656159808.103, "dur": 1.068,
    "args": {
      "External id": 2479193,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6896
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656159810.888, "dur": 2.420,
    "args": {
      "External id": 2479194,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6897
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656159814.761, "dur": 0.652,
    "args": {
      "External id": 2479195,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6898
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656159816.999, "dur": 0.785,
    "args": {
      "External id": 2479196,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6899
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656159837.667, "dur": 16527.975,
    "args": {
      "External id": 2479197,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6900
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656159853.874, "dur": 16504.701,
    "args": {
      "External id": 2479198,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6901
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656159873.816, "dur": 14.914,
    "args": {
      "External id": 2479199,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6902
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514656159891.523, "dur": 16434.813,
    "args": {
      "External id": 2479200,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 6903
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656159894.053, "dur": 16431.747,
    "args": {
      "External id": 2479201,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 6904
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656159899.196, "dur": 6.209,
    "args": {
      "External id": 2479202,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6905
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656159907.210, "dur": 16415.639,
    "args": {
      "External id": 2479203,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 6906
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656176549.396, "dur": 36.798,
    "args": {
      "External id": 2479204,"Sequence number": 24740443, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 6907
    }
  },
  {
    "ph": "s", "id": 205, "pid": 1336754, "tid": 1336754, "ts": 1514656176549.396,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336754, "tid": 1336754,
    "ts": 1514656176573.774, "dur": 7.501,
    "args": {
      "External id": 2479205,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 6908
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656176576.573, "dur": 4.350,
    "args": {
      "External id": 2479206,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 6909
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336754, "tid": 1336754,
    "ts": 1514656176650.914, "dur": 74.072,
    "args": {
      "External id": 2479207,"Record function id": 0, "Ev Idx": 6910
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656176726.425, "dur": 1059.049,
    "args": {
      "External id": 2479208,"Record function id": 0, "Ev Idx": 6911
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656176765.541, "dur": 1007.785,
    "args": {
      "External id": 2479209,"Sequence number": 24740444, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 6912
    }
  },
  {
    "ph": "s", "id": 204, "pid": 1336754, "tid": 1336754, "ts": 1514656176765.541,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514656176827.819, "dur": 40.155,
    "args": {
      "External id": 2479210,"kernel_hash": "caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/ao/caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6913
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656176880.536, "dur": 140.117,
    "args": {
      "External id": 2479211,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6914
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656177034.432, "dur": 45.944,
    "args": {
      "External id": 2479212,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6915
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656177089.240, "dur": 33.150,
    "args": {
      "External id": 2479213,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6916
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656177149.407, "dur": 26.321,
    "args": {
      "External id": 2479214,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6917
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656177192.961, "dur": 15.233,
    "args": {
      "External id": 2479215,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6918
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336754, "tid": 1336754,
    "ts": 1514656177226.417, "dur": 125.571,
    "args": {
      "External id": 2479216,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 6919
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656177275.104, "dur": 10.143,
    "args": {
      "External id": 2479217,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 6920
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656177279.678, "dur": 4.802,
    "args": {
      "External id": 2479218,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6921
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656177288.010, "dur": 5.806,
    "args": {
      "External id": 2479219,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6922
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656177295.142, "dur": 0.922,
    "args": {
      "External id": 2479220,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6923
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656177298.497, "dur": 5.304,
    "args": {
      "External id": 2479221,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6924
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656177363.202, "dur": 46.411,
    "args": {
      "External id": 2479222,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6925
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336754, "tid": 1336754,
    "ts": 1514656177438.162, "dur": 27.542,
    "args": {
      "External id": 2479223,"kernel_hash": "cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/wx/cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6926
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656177474.013, "dur": 42.039,
    "args": {
      "External id": 2479224,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6927
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656177525.338, "dur": 53.501,
    "args": {
      "External id": 2479225,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6928
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514656177601.499, "dur": 27.525,
    "args": {
      "External id": 2479226,"kernel_hash": "c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2f/c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 6929
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656177635.433, "dur": 35.500,
    "args": {
      "External id": 2479227,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 6930
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656177692.613, "dur": 17.478,
    "args": {
      "External id": 2479228,"kernel_hash": "cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/gi/cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 6931
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.11)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656177846.053, "dur": 73.147,
    "args": {
      "External id": 2479229,"Record function id": 0, "Ev Idx": 6932
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656178039.109, "dur": 52.750,
    "args": {
      "External id": 2479230,"Record function id": 0, "Ev Idx": 6933
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.12)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656178101.468, "dur": 17995.457,
    "args": {
      "External id": 2479231,"Record function id": 0, "Ev Idx": 6934
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.12)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656178110.562, "dur": 858.996,
    "args": {
      "External id": 2479232,"Record function id": 0, "Ev Idx": 6935
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656178193.112, "dur": 8.987,
    "args": {
      "External id": 2479233,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6936
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514656178218.368, "dur": 34.781,
    "args": {
      "External id": 2479234,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6937
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656178223.731, "dur": 2.241,
    "args": {
      "External id": 2479235,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6938
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656178229.679, "dur": 0.422,
    "args": {
      "External id": 2479236,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6939
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656178231.812, "dur": 0.487,
    "args": {
      "External id": 2479237,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6940
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656178233.536, "dur": 2.278,
    "args": {
      "External id": 2479238,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6941
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656178236.954, "dur": 0.447,
    "args": {
      "External id": 2479239,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6942
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656178238.661, "dur": 0.387,
    "args": {
      "External id": 2479240,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6943
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656178241.940, "dur": 1.647,
    "args": {
      "External id": 2479241,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6944
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656178244.927, "dur": 0.349,
    "args": {
      "External id": 2479242,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6945
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656178246.475, "dur": 0.425,
    "args": {
      "External id": 2479243,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6946
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656178264.419, "dur": 43.486,
    "args": {
      "External id": 2479244,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6947
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1336754,
    "ts": 1514656178340.978, "dur": 108.872,
    "args": {
      "External id": 2479245,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "1", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 6948
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656178353.231, "dur": 3.610,
    "args": {
      "External id": 2479246,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6949
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1336754,
    "ts": 1514656178361.600, "dur": 10.463,
    "args": {
      "External id": 2479247,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6950
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656178365.673, "dur": 5.990,
    "args": {
      "External id": 2479248,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "12846080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 6951
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656178369.087, "dur": 1.315,
    "args": {
      "External id": 2479249,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6952
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514656178380.039, "dur": 27.078,
    "args": {
      "External id": 2479250,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6953
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656178382.263, "dur": 0.482,
    "args": {
      "External id": 2479251,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6954
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656178384.214, "dur": 0.624,
    "args": {
      "External id": 2479252,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6423296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6955
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656178386.430, "dur": 0.361,
    "args": {
      "External id": 2479253,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6947584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6956
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656178389.487, "dur": 1.673,
    "args": {
      "External id": 2479254,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7471872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6957
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656178392.614, "dur": 0.420,
    "args": {
      "External id": 2479255,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7996160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6958
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656178394.583, "dur": 1.873,
    "args": {
      "External id": 2479256,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8520448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6959
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656178397.819, "dur": 0.261,
    "args": {
      "External id": 2479257,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "8520704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6960
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656178399.229, "dur": 0.348,
    "args": {
      "External id": 2479258,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "9962496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6961
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656178402.554, "dur": 0.191,
    "args": {
      "External id": 2479259,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11404288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6962
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656178417.545, "dur": 24.928,
    "args": {
      "External id": 2479260,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6963
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656178500.943, "dur": 377.709,
    "args": {
      "External id": 2479261,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 6964
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514656178547.271, "dur": 326.534,
    "args": {
      "External id": 2479262,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 6965, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1336754,
    "ts": 1514656178560.051, "dur": 308.000,
    "args": {
      "External id": 2479263,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 6966
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514656178901.484, "dur": 2.099,
    "args": {
      "External id": 2479264,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 6967, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.12)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656179027.167, "dur": 16855.252,
    "args": {
      "External id": 2479265,"Record function id": 0, "Ev Idx": 6968
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656179130.799, "dur": 6.813,
    "args": {
      "External id": 2479266,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 6969
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656179141.131, "dur": 1.116,
    "args": {
      "External id": 2479267,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6970
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656179143.682, "dur": 2.097,
    "args": {
      "External id": 2479268,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6971
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656179147.572, "dur": 0.872,
    "args": {
      "External id": 2479269,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6972
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656179149.688, "dur": 0.910,
    "args": {
      "External id": 2479270,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6973
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656179151.822, "dur": 0.973,
    "args": {
      "External id": 2479271,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6974
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656179155.842, "dur": 0.884,
    "args": {
      "External id": 2479272,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6975
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656179158.091, "dur": 1.764,
    "args": {
      "External id": 2479273,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6976
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656179161.388, "dur": 0.679,
    "args": {
      "External id": 2479274,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6977
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656179163.427, "dur": 0.580,
    "args": {
      "External id": 2479275,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6978
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656179183.970, "dur": 16661.666,
    "args": {
      "External id": 2479276,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6979
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656179198.976, "dur": 16640.119,
    "args": {
      "External id": 2479277,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6980
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656179221.255, "dur": 13.847,
    "args": {
      "External id": 2479278,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6981
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514656179238.183, "dur": 16570.064,
    "args": {
      "External id": 2479279,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 6982
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656179240.746, "dur": 16566.964,
    "args": {
      "External id": 2479280,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 6983
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656179246.134, "dur": 5.680,
    "args": {
      "External id": 2479281,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6984
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656179253.269, "dur": 16551.526,
    "args": {
      "External id": 2479282,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 6985
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656196036.205, "dur": 35.397,
    "args": {
      "External id": 2479283,"Sequence number": 24740445, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 6986
    }
  },
  {
    "ph": "s", "id": 203, "pid": 1336754, "tid": 1336754, "ts": 1514656196036.205,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336754, "tid": 1336754,
    "ts": 1514656196058.987, "dur": 8.041,
    "args": {
      "External id": 2479284,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 6987
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656196062.419, "dur": 4.223,
    "args": {
      "External id": 2479285,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 6988
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336754, "tid": 1336754,
    "ts": 1514656196133.675, "dur": 75.487,
    "args": {
      "External id": 2479286,"Record function id": 0, "Ev Idx": 6989
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656196210.637, "dur": 1070.378,
    "args": {
      "External id": 2479287,"Record function id": 0, "Ev Idx": 6990
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656196249.907, "dur": 1018.411,
    "args": {
      "External id": 2479288,"Sequence number": 24740446, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 6991
    }
  },
  {
    "ph": "s", "id": 202, "pid": 1336754, "tid": 1336754, "ts": 1514656196249.907,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514656196313.443, "dur": 43.431,
    "args": {
      "External id": 2479289,"kernel_hash": "caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/ao/caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6992
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656196369.427, "dur": 103.721,
    "args": {
      "External id": 2479290,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6993
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656196483.003, "dur": 40.821,
    "args": {
      "External id": 2479291,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6994
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656196554.256, "dur": 36.736,
    "args": {
      "External id": 2479292,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6995
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656196617.686, "dur": 26.308,
    "args": {
      "External id": 2479293,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6996
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656196660.200, "dur": 14.401,
    "args": {
      "External id": 2479294,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6997
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336754, "tid": 1336754,
    "ts": 1514656196691.666, "dur": 123.942,
    "args": {
      "External id": 2479295,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 6998
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656196739.301, "dur": 10.610,
    "args": {
      "External id": 2479296,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 6999
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656196744.093, "dur": 5.083,
    "args": {
      "External id": 2479297,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7000
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656196752.513, "dur": 5.918,
    "args": {
      "External id": 2479298,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7001
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656196759.845, "dur": 0.798,
    "args": {
      "External id": 2479299,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7002
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656196763.169, "dur": 4.639,
    "args": {
      "External id": 2479300,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7003
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656196825.471, "dur": 48.235,
    "args": {
      "External id": 2479301,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7004
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336754, "tid": 1336754,
    "ts": 1514656196902.477, "dur": 27.909,
    "args": {
      "External id": 2479302,"kernel_hash": "cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/wx/cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7005
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656196939.607, "dur": 77.222,
    "args": {
      "External id": 2479303,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7006
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656197029.164, "dur": 41.618,
    "args": {
      "External id": 2479304,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7007
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514656197097.711, "dur": 27.988,
    "args": {
      "External id": 2479305,"kernel_hash": "c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2f/c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 7008
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656197133.062, "dur": 35.551,
    "args": {
      "External id": 2479306,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 7009
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656197187.727, "dur": 17.408,
    "args": {
      "External id": 2479307,"kernel_hash": "cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/gi/cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 7010
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.12)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656197341.939, "dur": 72.274,
    "args": {
      "External id": 2479308,"Record function id": 0, "Ev Idx": 7011
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656197483.459, "dur": 63.254,
    "args": {
      "External id": 2479309,"Record function id": 0, "Ev Idx": 7012
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.13)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656197558.488, "dur": 18001.017,
    "args": {
      "External id": 2479310,"Record function id": 0, "Ev Idx": 7013
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.13)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656197567.414, "dur": 874.022,
    "args": {
      "External id": 2479311,"Record function id": 0, "Ev Idx": 7014
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656197647.621, "dur": 8.612,
    "args": {
      "External id": 2479312,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7015
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514656197670.335, "dur": 36.001,
    "args": {
      "External id": 2479313,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7016
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656197675.855, "dur": 2.311,
    "args": {
      "External id": 2479314,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7017
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656197682.230, "dur": 0.489,
    "args": {
      "External id": 2479315,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7018
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656197684.104, "dur": 0.279,
    "args": {
      "External id": 2479316,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7019
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656197685.686, "dur": 0.623,
    "args": {
      "External id": 2479317,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7020
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656197689.288, "dur": 0.458,
    "args": {
      "External id": 2479318,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7021
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656197691.201, "dur": 0.299,
    "args": {
      "External id": 2479319,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7022
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656197692.890, "dur": 3.411,
    "args": {
      "External id": 2479320,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7023
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656197697.698, "dur": 0.447,
    "args": {
      "External id": 2479321,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7024
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656197699.415, "dur": 0.441,
    "args": {
      "External id": 2479322,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7025
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656197717.176, "dur": 41.540,
    "args": {
      "External id": 2479323,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7026
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1336754,
    "ts": 1514656197790.990, "dur": 110.388,
    "args": {
      "External id": 2479324,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "1", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 7027
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656197802.843, "dur": 3.473,
    "args": {
      "External id": 2479325,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7028
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1336754,
    "ts": 1514656197810.884, "dur": 10.110,
    "args": {
      "External id": 2479326,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7029
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656197815.103, "dur": 5.511,
    "args": {
      "External id": 2479327,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "12846080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 7030
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656197818.761, "dur": 0.565,
    "args": {
      "External id": 2479328,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7031
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514656197828.420, "dur": 28.800,
    "args": {
      "External id": 2479329,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7032
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656197830.634, "dur": 2.366,
    "args": {
      "External id": 2479330,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7033
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656197834.884, "dur": 0.210,
    "args": {
      "External id": 2479331,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6423296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7034
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656197836.702, "dur": 0.325,
    "args": {
      "External id": 2479332,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6947584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7035
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656197840.078, "dur": 1.551,
    "args": {
      "External id": 2479333,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7471872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7036
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656197842.899, "dur": 0.303,
    "args": {
      "External id": 2479334,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7996160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7037
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656197844.475, "dur": 0.303,
    "args": {
      "External id": 2479335,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8520448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7038
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656197847.760, "dur": 0.169,
    "args": {
      "External id": 2479336,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "8520704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7039
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656197849.309, "dur": 0.275,
    "args": {
      "External id": 2479337,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "9962496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7040
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656197850.785, "dur": 2.147,
    "args": {
      "External id": 2479338,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11404288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7041
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656197869.247, "dur": 24.457,
    "args": {
      "External id": 2479339,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7042
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656197952.203, "dur": 397.480,
    "args": {
      "External id": 2479340,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 7043
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514656198020.193, "dur": 324.381,
    "args": {
      "External id": 2479341,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 7044, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1336754,
    "ts": 1514656198032.375, "dur": 307.028,
    "args": {
      "External id": 2479342,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 7045
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514656198374.335, "dur": 2.034,
    "args": {
      "External id": 2479343,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 7046, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.13)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656198461.359, "dur": 16890.780,
    "args": {
      "External id": 2479344,"Record function id": 0, "Ev Idx": 7047
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656198574.463, "dur": 6.683,
    "args": {
      "External id": 2479345,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 7048
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656198584.993, "dur": 0.946,
    "args": {
      "External id": 2479346,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7049
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656198587.556, "dur": 2.155,
    "args": {
      "External id": 2479347,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7050
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656198591.358, "dur": 0.748,
    "args": {
      "External id": 2479348,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7051
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656198593.415, "dur": 1.301,
    "args": {
      "External id": 2479349,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7052
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656198595.854, "dur": 1.364,
    "args": {
      "External id": 2479350,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7053
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656198600.218, "dur": 1.059,
    "args": {
      "External id": 2479351,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7054
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656198602.906, "dur": 2.035,
    "args": {
      "External id": 2479352,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7055
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656198606.913, "dur": 0.674,
    "args": {
      "External id": 2479353,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7056
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656198609.164, "dur": 0.546,
    "args": {
      "External id": 2479354,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7057
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656198630.369, "dur": 16683.069,
    "args": {
      "External id": 2479355,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7058
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656198646.220, "dur": 16660.175,
    "args": {
      "External id": 2479356,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7059
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656198669.654, "dur": 13.319,
    "args": {
      "External id": 2479357,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7060
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514656198685.751, "dur": 16588.882,
    "args": {
      "External id": 2479358,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 7061
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656198688.444, "dur": 16585.726,
    "args": {
      "External id": 2479359,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 7062
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656198694.255, "dur": 4.835,
    "args": {
      "External id": 2479360,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7063
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656198700.657, "dur": 16570.588,
    "args": {
      "External id": 2479361,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 7064
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656215484.898, "dur": 35.448,
    "args": {
      "External id": 2479362,"Sequence number": 24740447, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 7065
    }
  },
  {
    "ph": "s", "id": 201, "pid": 1336754, "tid": 1336754, "ts": 1514656215484.898,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336754, "tid": 1336754,
    "ts": 1514656215508.439, "dur": 7.112,
    "args": {
      "External id": 2479363,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 7066
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656215511.276, "dur": 4.065,
    "args": {
      "External id": 2479364,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 7067
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336754, "tid": 1336754,
    "ts": 1514656215598.656, "dur": 75.252,
    "args": {
      "External id": 2479365,"Record function id": 0, "Ev Idx": 7068
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656215675.463, "dur": 1055.544,
    "args": {
      "External id": 2479366,"Record function id": 0, "Ev Idx": 7069
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656215715.068, "dur": 1002.829,
    "args": {
      "External id": 2479367,"Sequence number": 24740448, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 7070
    }
  },
  {
    "ph": "s", "id": 200, "pid": 1336754, "tid": 1336754, "ts": 1514656215715.068,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514656215780.292, "dur": 41.017,
    "args": {
      "External id": 2479368,"kernel_hash": "caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/ao/caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7071
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656215832.891, "dur": 104.389,
    "args": {
      "External id": 2479369,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7072
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656215948.226, "dur": 72.374,
    "args": {
      "External id": 2479370,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7073
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656216032.679, "dur": 36.691,
    "args": {
      "External id": 2479371,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7074
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656216094.721, "dur": 26.538,
    "args": {
      "External id": 2479372,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7075
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656216139.128, "dur": 15.493,
    "args": {
      "External id": 2479373,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7076
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336754, "tid": 1336754,
    "ts": 1514656216172.317, "dur": 125.875,
    "args": {
      "External id": 2479374,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 7077
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656216222.346, "dur": 11.460,
    "args": {
      "External id": 2479375,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 7078
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656216227.667, "dur": 5.094,
    "args": {
      "External id": 2479376,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7079
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656216236.460, "dur": 5.338,
    "args": {
      "External id": 2479377,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7080
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656216243.131, "dur": 0.974,
    "args": {
      "External id": 2479378,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7081
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656216246.506, "dur": 3.933,
    "args": {
      "External id": 2479379,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7082
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656216307.781, "dur": 46.971,
    "args": {
      "External id": 2479380,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7083
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336754, "tid": 1336754,
    "ts": 1514656216382.587, "dur": 26.583,
    "args": {
      "External id": 2479381,"kernel_hash": "cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/wx/cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7084
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656216418.160, "dur": 41.587,
    "args": {
      "External id": 2479382,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7085
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656216468.401, "dur": 35.091,
    "args": {
      "External id": 2479383,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7086
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514656216541.034, "dur": 27.822,
    "args": {
      "External id": 2479384,"kernel_hash": "c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2f/c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 7087
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656216576.572, "dur": 37.941,
    "args": {
      "External id": 2479385,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 7088
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656216635.026, "dur": 17.343,
    "args": {
      "External id": 2479386,"kernel_hash": "cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/gi/cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 7089
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.13)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656216790.624, "dur": 72.939,
    "args": {
      "External id": 2479387,"Record function id": 0, "Ev Idx": 7090
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656216932.136, "dur": 46.791,
    "args": {
      "External id": 2479388,"Record function id": 0, "Ev Idx": 7091
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.14)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656217026.236, "dur": 18064.845,
    "args": {
      "External id": 2479389,"Record function id": 0, "Ev Idx": 7092
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.14)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656217036.473, "dur": 868.769,
    "args": {
      "External id": 2479390,"Record function id": 0, "Ev Idx": 7093
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656217115.915, "dur": 7.835,
    "args": {
      "External id": 2479391,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7094
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514656217136.129, "dur": 42.859,
    "args": {
      "External id": 2479392,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7095
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656217145.518, "dur": 2.545,
    "args": {
      "External id": 2479393,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7096
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656217151.923, "dur": 0.387,
    "args": {
      "External id": 2479394,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7097
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656217153.671, "dur": 0.327,
    "args": {
      "External id": 2479395,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7098
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656217155.330, "dur": 1.932,
    "args": {
      "External id": 2479396,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7099
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656217158.540, "dur": 0.387,
    "args": {
      "External id": 2479397,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7100
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656217162.236, "dur": 0.335,
    "args": {
      "External id": 2479398,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7101
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656217166.078, "dur": 1.612,
    "args": {
      "External id": 2479399,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7102
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656217168.815, "dur": 0.308,
    "args": {
      "External id": 2479400,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7103
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656217170.506, "dur": 1.641,
    "args": {
      "External id": 2479401,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7104
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656217189.647, "dur": 41.987,
    "args": {
      "External id": 2479402,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7105
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1336754,
    "ts": 1514656217264.848, "dur": 107.858,
    "args": {
      "External id": 2479403,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "1", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 7106
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656217275.573, "dur": 3.949,
    "args": {
      "External id": 2479404,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7107
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1336754,
    "ts": 1514656217284.664, "dur": 9.847,
    "args": {
      "External id": 2479405,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7108
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656217288.846, "dur": 5.266,
    "args": {
      "External id": 2479406,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "12846080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 7109
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656217292.177, "dur": 0.622,
    "args": {
      "External id": 2479407,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7110
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514656217301.655, "dur": 27.472,
    "args": {
      "External id": 2479408,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7111
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656217303.986, "dur": 0.503,
    "args": {
      "External id": 2479409,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7112
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656217305.880, "dur": 0.553,
    "args": {
      "External id": 2479410,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6423296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7113
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656217308.058, "dur": 2.194,
    "args": {
      "External id": 2479411,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6947584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7114
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656217312.003, "dur": 1.807,
    "args": {
      "External id": 2479412,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7471872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7115
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656217315.300, "dur": 0.492,
    "args": {
      "External id": 2479413,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7996160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7116
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656217318.826, "dur": 0.303,
    "args": {
      "External id": 2479414,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8520448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7117
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656217320.465, "dur": 0.168,
    "args": {
      "External id": 2479415,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "8520704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7118
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656217321.916, "dur": 0.306,
    "args": {
      "External id": 2479416,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "9962496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7119
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656217325.097, "dur": 0.243,
    "args": {
      "External id": 2479417,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11404288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7120
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656217339.933, "dur": 25.042,
    "args": {
      "External id": 2479418,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7121
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656217426.108, "dur": 379.203,
    "args": {
      "External id": 2479419,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 7122
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514656217456.635, "dur": 343.305,
    "args": {
      "External id": 2479420,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 7123, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1336754,
    "ts": 1514656217466.388, "dur": 327.318,
    "args": {
      "External id": 2479421,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 7124
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514656217830.525, "dur": 2.853,
    "args": {
      "External id": 2479422,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 7125, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.14)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656217927.149, "dur": 16931.948,
    "args": {
      "External id": 2479423,"Record function id": 0, "Ev Idx": 7126
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656218074.298, "dur": 6.674,
    "args": {
      "External id": 2479424,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 7127
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656218084.870, "dur": 1.405,
    "args": {
      "External id": 2479425,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7128
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656218087.868, "dur": 2.119,
    "args": {
      "External id": 2479426,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7129
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656218091.870, "dur": 1.000,
    "args": {
      "External id": 2479427,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7130
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656218094.301, "dur": 0.850,
    "args": {
      "External id": 2479428,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7131
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656218096.321, "dur": 0.885,
    "args": {
      "External id": 2479429,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7132
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656218100.461, "dur": 0.640,
    "args": {
      "External id": 2479430,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7133
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656218102.798, "dur": 2.112,
    "args": {
      "External id": 2479431,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7134
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656218106.462, "dur": 0.905,
    "args": {
      "External id": 2479432,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7135
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656218108.888, "dur": 0.665,
    "args": {
      "External id": 2479433,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7136
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656218130.660, "dur": 16689.524,
    "args": {
      "External id": 2479434,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7137
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656218146.579, "dur": 16666.758,
    "args": {
      "External id": 2479435,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7138
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656218169.401, "dur": 14.883,
    "args": {
      "External id": 2479436,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7139
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514656218187.126, "dur": 16594.896,
    "args": {
      "External id": 2479437,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 7140
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656218189.638, "dur": 16591.825,
    "args": {
      "External id": 2479438,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 7141
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656218195.108, "dur": 5.540,
    "args": {
      "External id": 2479439,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7142
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656218202.356, "dur": 16576.236,
    "args": {
      "External id": 2479440,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 7143
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656235028.631, "dur": 34.640,
    "args": {
      "External id": 2479441,"Sequence number": 24740449, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 7144
    }
  },
  {
    "ph": "s", "id": 199, "pid": 1336754, "tid": 1336754, "ts": 1514656235028.631,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336754, "tid": 1336754,
    "ts": 1514656235051.190, "dur": 7.278,
    "args": {
      "External id": 2479442,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 7145
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656235053.862, "dur": 4.261,
    "args": {
      "External id": 2479443,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 7146
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336754, "tid": 1336754,
    "ts": 1514656235130.424, "dur": 75.446,
    "args": {
      "External id": 2479444,"Record function id": 0, "Ev Idx": 7147
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656235207.271, "dur": 1063.653,
    "args": {
      "External id": 2479445,"Record function id": 0, "Ev Idx": 7148
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656235245.277, "dur": 1012.044,
    "args": {
      "External id": 2479446,"Sequence number": 24740450, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 7149
    }
  },
  {
    "ph": "s", "id": 198, "pid": 1336754, "tid": 1336754, "ts": 1514656235245.277,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514656235308.704, "dur": 42.491,
    "args": {
      "External id": 2479447,"kernel_hash": "caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/ao/caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7150
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656235362.966, "dur": 105.929,
    "args": {
      "External id": 2479448,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7151
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656235478.031, "dur": 39.352,
    "args": {
      "External id": 2479449,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7152
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656235525.792, "dur": 52.084,
    "args": {
      "External id": 2479450,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7153
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656235605.808, "dur": 26.186,
    "args": {
      "External id": 2479451,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7154
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656235648.374, "dur": 14.812,
    "args": {
      "External id": 2479452,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7155
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336754, "tid": 1336754,
    "ts": 1514656235681.116, "dur": 128.805,
    "args": {
      "External id": 2479453,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 7156
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656235733.647, "dur": 11.773,
    "args": {
      "External id": 2479454,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 7157
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656235739.153, "dur": 5.373,
    "args": {
      "External id": 2479455,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7158
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656235748.057, "dur": 6.008,
    "args": {
      "External id": 2479456,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7159
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656235755.288, "dur": 1.192,
    "args": {
      "External id": 2479457,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7160
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656235758.652, "dur": 3.823,
    "args": {
      "External id": 2479458,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7161
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656235820.684, "dur": 46.564,
    "args": {
      "External id": 2479459,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7162
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336754, "tid": 1336754,
    "ts": 1514656235898.111, "dur": 27.489,
    "args": {
      "External id": 2479460,"kernel_hash": "cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/wx/cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7163
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656235934.664, "dur": 42.654,
    "args": {
      "External id": 2479461,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7164
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656236022.216, "dur": 41.499,
    "args": {
      "External id": 2479462,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7165
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514656236087.331, "dur": 26.144,
    "args": {
      "External id": 2479463,"kernel_hash": "c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2f/c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 7166
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656236119.188, "dur": 35.313,
    "args": {
      "External id": 2479464,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 7167
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656236175.727, "dur": 17.392,
    "args": {
      "External id": 2479465,"kernel_hash": "cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/gi/cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 7168
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.14)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656236332.093, "dur": 73.494,
    "args": {
      "External id": 2479466,"Record function id": 0, "Ev Idx": 7169
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656236474.097, "dur": 45.793,
    "args": {
      "External id": 2479467,"Record function id": 0, "Ev Idx": 7170
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.15)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656236548.000, "dur": 18325.384,
    "args": {
      "External id": 2479468,"Record function id": 0, "Ev Idx": 7171
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.15)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656236558.148, "dur": 916.004,
    "args": {
      "External id": 2479469,"Record function id": 0, "Ev Idx": 7172
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656236639.442, "dur": 9.189,
    "args": {
      "External id": 2479470,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7173
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514656236662.666, "dur": 35.886,
    "args": {
      "External id": 2479471,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7174
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656236667.776, "dur": 2.308,
    "args": {
      "External id": 2479472,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7175
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656236674.463, "dur": 0.390,
    "args": {
      "External id": 2479473,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7176
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656236676.036, "dur": 0.402,
    "args": {
      "External id": 2479474,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7177
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656236678.169, "dur": 0.543,
    "args": {
      "External id": 2479475,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7178
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656236681.773, "dur": 0.458,
    "args": {
      "External id": 2479476,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7179
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656236683.369, "dur": 0.294,
    "args": {
      "External id": 2479477,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7180
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656236684.921, "dur": 3.561,
    "args": {
      "External id": 2479478,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7181
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656236690.046, "dur": 0.521,
    "args": {
      "External id": 2479479,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7182
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656236691.916, "dur": 0.289,
    "args": {
      "External id": 2479480,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7183
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656236709.582, "dur": 44.344,
    "args": {
      "External id": 2479481,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7184
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1336754,
    "ts": 1514656236786.367, "dur": 119.061,
    "args": {
      "External id": 2479482,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "1", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 7185
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656236796.982, "dur": 4.006,
    "args": {
      "External id": 2479483,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7186
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1336754,
    "ts": 1514656236806.047, "dur": 10.427,
    "args": {
      "External id": 2479484,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7187
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656236810.434, "dur": 5.622,
    "args": {
      "External id": 2479485,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "12846080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 7188
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656236814.220, "dur": 0.519,
    "args": {
      "External id": 2479486,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7189
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514656236823.823, "dur": 29.236,
    "args": {
      "External id": 2479487,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7190
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656236825.768, "dur": 2.467,
    "args": {
      "External id": 2479488,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7191
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656236829.878, "dur": 0.214,
    "args": {
      "External id": 2479489,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6423296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7192
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656236831.699, "dur": 0.376,
    "args": {
      "External id": 2479490,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6947584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7193
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656236835.455, "dur": 1.460,
    "args": {
      "External id": 2479491,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7471872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7194
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656236838.059, "dur": 0.256,
    "args": {
      "External id": 2479492,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7996160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7195
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656236840.073, "dur": 0.286,
    "args": {
      "External id": 2479493,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8520448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7196
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656236842.979, "dur": 0.191,
    "args": {
      "External id": 2479494,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "8520704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7197
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656236844.694, "dur": 0.233,
    "args": {
      "External id": 2479495,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "9962496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7198
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656236846.194, "dur": 2.055,
    "args": {
      "External id": 2479496,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11404288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7199
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656236872.008, "dur": 25.409,
    "args": {
      "External id": 2479497,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7200
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656236957.294, "dur": 418.066,
    "args": {
      "External id": 2479498,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 7201
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514656237026.202, "dur": 343.762,
    "args": {
      "External id": 2479499,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 7202, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1336754,
    "ts": 1514656237039.951, "dur": 323.982,
    "args": {
      "External id": 2479500,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 7203
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514656237400.547, "dur": 2.586,
    "args": {
      "External id": 2479501,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 7204, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.15)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656237494.826, "dur": 17188.674,
    "args": {
      "External id": 2479502,"Record function id": 0, "Ev Idx": 7205
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656237612.231, "dur": 7.094,
    "args": {
      "External id": 2479503,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 7206
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656237622.949, "dur": 1.198,
    "args": {
      "External id": 2479504,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7207
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656237625.747, "dur": 2.138,
    "args": {
      "External id": 2479505,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7208
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656237629.625, "dur": 0.978,
    "args": {
      "External id": 2479506,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7209
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656237632.076, "dur": 0.875,
    "args": {
      "External id": 2479507,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7210
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656237634.096, "dur": 0.842,
    "args": {
      "External id": 2479508,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7211
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656237638.907, "dur": 0.961,
    "args": {
      "External id": 2479509,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7212
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656237641.266, "dur": 1.994,
    "args": {
      "External id": 2479510,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7213
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656237644.602, "dur": 0.819,
    "args": {
      "External id": 2479511,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7214
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656237646.976, "dur": 0.947,
    "args": {
      "External id": 2479512,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7215
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656237668.380, "dur": 16976.223,
    "args": {
      "External id": 2479513,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7216
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656237684.270, "dur": 16953.091,
    "args": {
      "External id": 2479514,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7217
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656237705.416, "dur": 13.878,
    "args": {
      "External id": 2479515,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7218
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514656237722.089, "dur": 16883.668,
    "args": {
      "External id": 2479516,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 7219
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656237724.333, "dur": 16880.918,
    "args": {
      "External id": 2479517,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 7220
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656237729.578, "dur": 4.982,
    "args": {
      "External id": 2479518,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7221
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656237736.270, "dur": 16865.827,
    "args": {
      "External id": 2479519,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 7222
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656254813.271, "dur": 36.283,
    "args": {
      "External id": 2479520,"Sequence number": 24740451, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 7223
    }
  },
  {
    "ph": "s", "id": 197, "pid": 1336754, "tid": 1336754, "ts": 1514656254813.271,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336754, "tid": 1336754,
    "ts": 1514656254837.189, "dur": 7.610,
    "args": {
      "External id": 2479521,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 7224
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656254840.266, "dur": 4.327,
    "args": {
      "External id": 2479522,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 7225
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336754, "tid": 1336754,
    "ts": 1514656254910.148, "dur": 103.233,
    "args": {
      "External id": 2479523,"Record function id": 0, "Ev Idx": 7226
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656255016.875, "dur": 1064.397,
    "args": {
      "External id": 2479524,"Record function id": 0, "Ev Idx": 7227
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656255060.094, "dur": 1006.974,
    "args": {
      "External id": 2479525,"Sequence number": 24740452, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 7228
    }
  },
  {
    "ph": "s", "id": 196, "pid": 1336754, "tid": 1336754, "ts": 1514656255060.094,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514656255126.907, "dur": 43.226,
    "args": {
      "External id": 2479526,"kernel_hash": "caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/ao/caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7229
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656255182.531, "dur": 106.460,
    "args": {
      "External id": 2479527,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7230
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656255298.900, "dur": 39.203,
    "args": {
      "External id": 2479528,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7231
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656255346.847, "dur": 32.196,
    "args": {
      "External id": 2479529,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7232
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656255402.871, "dur": 24.803,
    "args": {
      "External id": 2479530,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7233
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656255445.335, "dur": 14.714,
    "args": {
      "External id": 2479531,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7234
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336754, "tid": 1336754,
    "ts": 1514656255476.933, "dur": 143.985,
    "args": {
      "External id": 2479532,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 7235
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656255525.181, "dur": 28.293,
    "args": {
      "External id": 2479533,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 7236
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656255545.890, "dur": 6.486,
    "args": {
      "External id": 2479534,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7237
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656255556.451, "dur": 6.009,
    "args": {
      "External id": 2479535,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7238
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656255563.659, "dur": 1.162,
    "args": {
      "External id": 2479536,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7239
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656255567.457, "dur": 3.418,
    "args": {
      "External id": 2479537,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7240
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656255632.432, "dur": 51.409,
    "args": {
      "External id": 2479538,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7241
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336754, "tid": 1336754,
    "ts": 1514656255713.768, "dur": 27.351,
    "args": {
      "External id": 2479539,"kernel_hash": "cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/wx/cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7242
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656255750.381, "dur": 42.890,
    "args": {
      "External id": 2479540,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7243
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656255801.896, "dur": 35.837,
    "args": {
      "External id": 2479541,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7244
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514656255859.522, "dur": 26.754,
    "args": {
      "External id": 2479542,"kernel_hash": "c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2f/c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 7245
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656255891.795, "dur": 34.750,
    "args": {
      "External id": 2479543,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 7246
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656255946.064, "dur": 16.429,
    "args": {
      "External id": 2479544,"kernel_hash": "cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/gi/cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 7247
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.15)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656256143.937, "dur": 73.489,
    "args": {
      "External id": 2479545,"Record function id": 0, "Ev Idx": 7248
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656256287.968, "dur": 45.819,
    "args": {
      "External id": 2479546,"Record function id": 0, "Ev Idx": 7249
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.16)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656256342.596, "dur": 18147.998,
    "args": {
      "External id": 2479547,"Record function id": 0, "Ev Idx": 7250
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.16)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656256352.191, "dur": 932.826,
    "args": {
      "External id": 2479548,"Record function id": 0, "Ev Idx": 7251
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656256433.600, "dur": 7.772,
    "args": {
      "External id": 2479549,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7252
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514656256455.402, "dur": 35.751,
    "args": {
      "External id": 2479550,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7253
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656256460.595, "dur": 2.314,
    "args": {
      "External id": 2479551,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7254
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656256467.775, "dur": 0.419,
    "args": {
      "External id": 2479552,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7255
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656256469.527, "dur": 0.304,
    "args": {
      "External id": 2479553,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7256
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656256471.321, "dur": 0.295,
    "args": {
      "External id": 2479554,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7257
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656256474.562, "dur": 0.349,
    "args": {
      "External id": 2479555,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7258
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656256476.353, "dur": 0.389,
    "args": {
      "External id": 2479556,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7259
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656256478.092, "dur": 3.209,
    "args": {
      "External id": 2479557,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7260
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656256482.857, "dur": 0.328,
    "args": {
      "External id": 2479558,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7261
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656256484.424, "dur": 0.379,
    "args": {
      "External id": 2479559,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7262
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656256505.208, "dur": 57.473,
    "args": {
      "External id": 2479560,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7263
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1336754,
    "ts": 1514656256598.965, "dur": 112.888,
    "args": {
      "External id": 2479561,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "1", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 7264
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656256610.168, "dur": 6.372,
    "args": {
      "External id": 2479562,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7265
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1336754,
    "ts": 1514656256621.630, "dur": 10.706,
    "args": {
      "External id": 2479563,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7266
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656256625.744, "dur": 6.160,
    "args": {
      "External id": 2479564,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "12846080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 7267
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656256629.377, "dur": 0.897,
    "args": {
      "External id": 2479565,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7268
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514656256640.224, "dur": 25.722,
    "args": {
      "External id": 2479566,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7269
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656256642.279, "dur": 0.257,
    "args": {
      "External id": 2479567,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7270
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656256644.267, "dur": 1.956,
    "args": {
      "External id": 2479568,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6423296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7271
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656256647.703, "dur": 0.212,
    "args": {
      "External id": 2479569,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6947584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7272
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656256649.465, "dur": 1.782,
    "args": {
      "External id": 2479570,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7471872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7273
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656256653.899, "dur": 0.194,
    "args": {
      "External id": 2479571,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7996160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7274
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656256655.422, "dur": 0.281,
    "args": {
      "External id": 2479572,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8520448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7275
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656256656.872, "dur": 0.245,
    "args": {
      "External id": 2479573,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "8520704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7276
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656256659.950, "dur": 0.339,
    "args": {
      "External id": 2479574,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "9962496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7277
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656256661.575, "dur": 0.392,
    "args": {
      "External id": 2479575,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11404288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7278
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656256677.225, "dur": 26.654,
    "args": {
      "External id": 2479576,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7279
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656256765.477, "dur": 420.545,
    "args": {
      "External id": 2479577,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 7280
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514656256795.570, "dur": 385.191,
    "args": {
      "External id": 2479578,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 7281, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1336754,
    "ts": 1514656256805.580, "dur": 368.470,
    "args": {
      "External id": 2479579,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 7282
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514656257212.703, "dur": 2.293,
    "args": {
      "External id": 2479580,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 7283, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.16)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656257306.417, "dur": 16993.929,
    "args": {
      "External id": 2479581,"Record function id": 0, "Ev Idx": 7284
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656257404.701, "dur": 5.876,
    "args": {
      "External id": 2479582,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 7285
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656257413.939, "dur": 1.458,
    "args": {
      "External id": 2479583,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7286
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656257416.946, "dur": 2.106,
    "args": {
      "External id": 2479584,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7287
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656257421.008, "dur": 0.845,
    "args": {
      "External id": 2479585,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7288
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656257423.211, "dur": 0.823,
    "args": {
      "External id": 2479586,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7289
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656257425.261, "dur": 0.782,
    "args": {
      "External id": 2479587,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7290
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656257429.185, "dur": 0.718,
    "args": {
      "External id": 2479588,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7291
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656257431.312, "dur": 1.890,
    "args": {
      "External id": 2479589,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7292
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656257434.616, "dur": 0.774,
    "args": {
      "External id": 2479590,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7293
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656257437.089, "dur": 0.746,
    "args": {
      "External id": 2479591,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7294
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656257456.836, "dur": 16805.631,
    "args": {
      "External id": 2479592,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7295
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656257472.037, "dur": 16783.729,
    "args": {
      "External id": 2479593,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7296
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656257494.365, "dur": 14.737,
    "args": {
      "External id": 2479594,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7297
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514656257511.974, "dur": 16713.009,
    "args": {
      "External id": 2479595,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 7298
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656257514.567, "dur": 16709.881,
    "args": {
      "External id": 2479596,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 7299
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656257519.991, "dur": 5.484,
    "args": {
      "External id": 2479597,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7300
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656257543.028, "dur": 16678.456,
    "args": {
      "External id": 2479598,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 7301
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656274433.324, "dur": 33.751,
    "args": {
      "External id": 2479599,"Sequence number": 24740453, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 7302
    }
  },
  {
    "ph": "s", "id": 195, "pid": 1336754, "tid": 1336754, "ts": 1514656274433.324,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336754, "tid": 1336754,
    "ts": 1514656274455.306, "dur": 7.036,
    "args": {
      "External id": 2479600,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 7303
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656274458.336, "dur": 3.791,
    "args": {
      "External id": 2479601,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 7304
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336754, "tid": 1336754,
    "ts": 1514656274538.231, "dur": 77.476,
    "args": {
      "External id": 2479602,"Record function id": 0, "Ev Idx": 7305
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656274618.358, "dur": 1094.296,
    "args": {
      "External id": 2479603,"Record function id": 0, "Ev Idx": 7306
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656274659.182, "dur": 1039.810,
    "args": {
      "External id": 2479604,"Sequence number": 24740454, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 7307
    }
  },
  {
    "ph": "s", "id": 194, "pid": 1336754, "tid": 1336754, "ts": 1514656274659.182,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514656274724.066, "dur": 42.499,
    "args": {
      "External id": 2479605,"kernel_hash": "caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/ao/caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7308
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656274778.457, "dur": 103.411,
    "args": {
      "External id": 2479606,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7309
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656274891.720, "dur": 40.112,
    "args": {
      "External id": 2479607,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7310
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656274940.257, "dur": 31.882,
    "args": {
      "External id": 2479608,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7311
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656275038.013, "dur": 29.011,
    "args": {
      "External id": 2479609,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7312
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656275084.385, "dur": 14.832,
    "args": {
      "External id": 2479610,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7313
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336754, "tid": 1336754,
    "ts": 1514656275118.142, "dur": 126.592,
    "args": {
      "External id": 2479611,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 7314
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656275166.430, "dur": 11.723,
    "args": {
      "External id": 2479612,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 7315
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656275171.293, "dur": 5.952,
    "args": {
      "External id": 2479613,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7316
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656275180.725, "dur": 6.368,
    "args": {
      "External id": 2479614,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7317
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656275188.334, "dur": 1.105,
    "args": {
      "External id": 2479615,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7318
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656275191.783, "dur": 5.643,
    "args": {
      "External id": 2479616,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7319
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656275276.822, "dur": 53.717,
    "args": {
      "External id": 2479617,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7320
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336754, "tid": 1336754,
    "ts": 1514656275363.177, "dur": 28.906,
    "args": {
      "External id": 2479618,"kernel_hash": "cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/wx/cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7321
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656275400.410, "dur": 41.815,
    "args": {
      "External id": 2479619,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7322
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656275450.763, "dur": 35.102,
    "args": {
      "External id": 2479620,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7323
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514656275506.311, "dur": 42.588,
    "args": {
      "External id": 2479621,"kernel_hash": "c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2f/c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 7324
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656275556.302, "dur": 38.631,
    "args": {
      "External id": 2479622,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 7325
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656275616.956, "dur": 18.505,
    "args": {
      "External id": 2479623,"kernel_hash": "cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/gi/cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 7326
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.16)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656275774.178, "dur": 76.794,
    "args": {
      "External id": 2479624,"Record function id": 0, "Ev Idx": 7327
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656275922.210, "dur": 46.170,
    "args": {
      "External id": 2479625,"Record function id": 0, "Ev Idx": 7328
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.17)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656275977.847, "dur": 18195.867,
    "args": {
      "External id": 2479626,"Record function id": 0, "Ev Idx": 7329
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.17)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656276025.726, "dur": 856.297,
    "args": {
      "External id": 2479627,"Record function id": 0, "Ev Idx": 7330
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656276108.148, "dur": 8.567,
    "args": {
      "External id": 2479628,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7331
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514656276130.759, "dur": 33.718,
    "args": {
      "External id": 2479629,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7332
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656276136.147, "dur": 2.200,
    "args": {
      "External id": 2479630,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7333
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656276142.148, "dur": 0.446,
    "args": {
      "External id": 2479631,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7334
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656276144.076, "dur": 0.530,
    "args": {
      "External id": 2479632,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7335
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656276145.888, "dur": 0.347,
    "args": {
      "External id": 2479633,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7336
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656276148.947, "dur": 0.486,
    "args": {
      "External id": 2479634,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7337
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656276150.897, "dur": 0.245,
    "args": {
      "External id": 2479635,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7338
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656276152.499, "dur": 3.251,
    "args": {
      "External id": 2479636,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7339
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656276156.952, "dur": 0.176,
    "args": {
      "External id": 2479637,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7340
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656276158.543, "dur": 0.154,
    "args": {
      "External id": 2479638,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7341
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656276176.545, "dur": 42.277,
    "args": {
      "External id": 2479639,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7342
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1336754,
    "ts": 1514656276251.235, "dur": 109.787,
    "args": {
      "External id": 2479640,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "1", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 7343
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656276261.653, "dur": 4.033,
    "args": {
      "External id": 2479641,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7344
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1336754,
    "ts": 1514656276270.489, "dur": 9.943,
    "args": {
      "External id": 2479642,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7345
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656276274.713, "dur": 5.309,
    "args": {
      "External id": 2479643,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "12846080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 7346
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656276278.148, "dur": 0.638,
    "args": {
      "External id": 2479644,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7347
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514656276287.366, "dur": 28.407,
    "args": {
      "External id": 2479645,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7348
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656276289.445, "dur": 1.987,
    "args": {
      "External id": 2479646,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7349
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656276293.247, "dur": 0.318,
    "args": {
      "External id": 2479647,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6423296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7350
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656276294.901, "dur": 0.515,
    "args": {
      "External id": 2479648,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6947584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7351
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656276298.667, "dur": 1.727,
    "args": {
      "External id": 2479649,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7471872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7352
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656276302.054, "dur": 0.293,
    "args": {
      "External id": 2479650,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7996160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7353
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656276303.694, "dur": 0.164,
    "args": {
      "External id": 2479651,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8520448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7354
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656276306.702, "dur": 0.287,
    "args": {
      "External id": 2479652,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "8520704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7355
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656276308.553, "dur": 0.476,
    "args": {
      "External id": 2479653,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "9962496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7356
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656276310.142, "dur": 1.654,
    "args": {
      "External id": 2479654,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11404288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7357
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656276328.489, "dur": 24.641,
    "args": {
      "External id": 2479655,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7358
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656276412.833, "dur": 372.152,
    "args": {
      "External id": 2479656,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 7359
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514656276443.355, "dur": 336.014,
    "args": {
      "External id": 2479657,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 7360, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1336754,
    "ts": 1514656276453.551, "dur": 319.435,
    "args": {
      "External id": 2479658,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 7361
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514656276810.404, "dur": 2.749,
    "args": {
      "External id": 2479659,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 7362, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.17)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656276902.088, "dur": 17048.471,
    "args": {
      "External id": 2479660,"Record function id": 0, "Ev Idx": 7363
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656277037.750, "dur": 6.877,
    "args": {
      "External id": 2479661,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 7364
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656277048.708, "dur": 1.150,
    "args": {
      "External id": 2479662,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7365
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656277051.893, "dur": 2.446,
    "args": {
      "External id": 2479663,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7366
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656277056.125, "dur": 0.779,
    "args": {
      "External id": 2479664,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7367
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656277058.479, "dur": 0.689,
    "args": {
      "External id": 2479665,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7368
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656277060.504, "dur": 0.781,
    "args": {
      "External id": 2479666,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7369
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656277064.541, "dur": 0.790,
    "args": {
      "External id": 2479667,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7370
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656277066.933, "dur": 1.709,
    "args": {
      "External id": 2479668,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7371
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656277070.102, "dur": 0.662,
    "args": {
      "External id": 2479669,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7372
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656277072.120, "dur": 0.698,
    "args": {
      "External id": 2479670,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7373
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656277094.299, "dur": 16816.478,
    "args": {
      "External id": 2479671,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7374
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656277110.872, "dur": 16792.530,
    "args": {
      "External id": 2479672,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7375
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656277135.500, "dur": 14.188,
    "args": {
      "External id": 2479673,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7376
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514656277152.528, "dur": 16719.344,
    "args": {
      "External id": 2479674,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 7377
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656277155.010, "dur": 16716.342,
    "args": {
      "External id": 2479675,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 7378
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656277160.225, "dur": 5.691,
    "args": {
      "External id": 2479676,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7379
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656277167.492, "dur": 16700.867,
    "args": {
      "External id": 2479677,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 7380
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656294112.951, "dur": 35.018,
    "args": {
      "External id": 2479678,"Sequence number": 24740455, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 7381
    }
  },
  {
    "ph": "s", "id": 193, "pid": 1336754, "tid": 1336754, "ts": 1514656294112.951,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336754, "tid": 1336754,
    "ts": 1514656294135.466, "dur": 7.747,
    "args": {
      "External id": 2479679,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 7382
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656294138.593, "dur": 4.238,
    "args": {
      "External id": 2479680,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 7383
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336754, "tid": 1336754,
    "ts": 1514656294211.844, "dur": 76.250,
    "args": {
      "External id": 2479681,"Record function id": 0, "Ev Idx": 7384
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656294289.605, "dur": 1051.991,
    "args": {
      "External id": 2479682,"Record function id": 0, "Ev Idx": 7385
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656294329.024, "dur": 999.176,
    "args": {
      "External id": 2479683,"Sequence number": 24740456, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 7386
    }
  },
  {
    "ph": "s", "id": 192, "pid": 1336754, "tid": 1336754, "ts": 1514656294329.024,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514656294392.616, "dur": 40.447,
    "args": {
      "External id": 2479684,"kernel_hash": "caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/ao/caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7387
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656294444.935, "dur": 110.033,
    "args": {
      "External id": 2479685,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7388
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656294568.460, "dur": 43.000,
    "args": {
      "External id": 2479686,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7389
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656294620.541, "dur": 32.963,
    "args": {
      "External id": 2479687,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7390
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656294679.838, "dur": 25.004,
    "args": {
      "External id": 2479688,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7391
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656294721.487, "dur": 14.718,
    "args": {
      "External id": 2479689,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7392
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336754, "tid": 1336754,
    "ts": 1514656294753.927, "dur": 124.466,
    "args": {
      "External id": 2479690,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 7393
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656294802.645, "dur": 10.886,
    "args": {
      "External id": 2479691,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 7394
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656294807.435, "dur": 5.342,
    "args": {
      "External id": 2479692,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7395
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656294816.193, "dur": 5.757,
    "args": {
      "External id": 2479693,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7396
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656294823.226, "dur": 1.113,
    "args": {
      "External id": 2479694,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7397
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656294826.604, "dur": 4.309,
    "args": {
      "External id": 2479695,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7398
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656294888.155, "dur": 45.602,
    "args": {
      "External id": 2479696,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7399
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336754, "tid": 1336754,
    "ts": 1514656294963.436, "dur": 65.630,
    "args": {
      "External id": 2479697,"kernel_hash": "cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/wx/cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7400
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656295042.641, "dur": 47.486,
    "args": {
      "External id": 2479698,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7401
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656295098.899, "dur": 35.148,
    "args": {
      "External id": 2479699,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7402
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514656295157.135, "dur": 26.301,
    "args": {
      "External id": 2479700,"kernel_hash": "c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2f/c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 7403
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656295189.283, "dur": 34.969,
    "args": {
      "External id": 2479701,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 7404
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656295244.161, "dur": 16.720,
    "args": {
      "External id": 2479702,"kernel_hash": "cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/gi/cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 7405
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.17)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656295403.005, "dur": 72.641,
    "args": {
      "External id": 2479703,"Record function id": 0, "Ev Idx": 7406
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656295563.288, "dur": 48.334,
    "args": {
      "External id": 2479704,"Record function id": 0, "Ev Idx": 7407
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.18)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656295621.450, "dur": 18024.300,
    "args": {
      "External id": 2479705,"Record function id": 0, "Ev Idx": 7408
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.18)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656295630.184, "dur": 867.631,
    "args": {
      "External id": 2479706,"Record function id": 0, "Ev Idx": 7409
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656295711.792, "dur": 7.987,
    "args": {
      "External id": 2479707,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7410
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514656295733.811, "dur": 34.467,
    "args": {
      "External id": 2479708,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7411
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656295739.309, "dur": 2.151,
    "args": {
      "External id": 2479709,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7412
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656295746.221, "dur": 0.236,
    "args": {
      "External id": 2479710,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7413
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656295747.707, "dur": 0.337,
    "args": {
      "External id": 2479711,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7414
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656295749.210, "dur": 0.327,
    "args": {
      "External id": 2479712,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7415
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656295752.325, "dur": 0.442,
    "args": {
      "External id": 2479713,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7416
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656295754.250, "dur": 0.378,
    "args": {
      "External id": 2479714,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7417
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656295755.680, "dur": 3.073,
    "args": {
      "External id": 2479715,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7418
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656295760.611, "dur": 0.274,
    "args": {
      "External id": 2479716,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7419
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656295762.106, "dur": 0.290,
    "args": {
      "External id": 2479717,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7420
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656295779.920, "dur": 40.735,
    "args": {
      "External id": 2479718,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7421
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1336754,
    "ts": 1514656295854.608, "dur": 108.625,
    "args": {
      "External id": 2479719,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "1", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 7422
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656295868.872, "dur": 5.230,
    "args": {
      "External id": 2479720,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7423
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1336754,
    "ts": 1514656295879.159, "dur": 9.700,
    "args": {
      "External id": 2479721,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7424
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656295883.352, "dur": 4.975,
    "args": {
      "External id": 2479722,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "12846080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 7425
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656295886.502, "dur": 0.549,
    "args": {
      "External id": 2479723,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7426
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514656295895.937, "dur": 26.428,
    "args": {
      "External id": 2479724,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7427
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656295898.268, "dur": 0.395,
    "args": {
      "External id": 2479725,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7428
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656295900.171, "dur": 1.941,
    "args": {
      "External id": 2479726,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6423296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7429
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656295903.303, "dur": 0.329,
    "args": {
      "External id": 2479727,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6947584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7430
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656295905.145, "dur": 1.478,
    "args": {
      "External id": 2479728,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7471872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7431
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656295909.621, "dur": 0.563,
    "args": {
      "External id": 2479729,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7996160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7432
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656295911.551, "dur": 0.278,
    "args": {
      "External id": 2479730,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8520448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7433
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656295912.916, "dur": 0.447,
    "args": {
      "External id": 2479731,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "8520704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7434
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656295916.321, "dur": 0.441,
    "args": {
      "External id": 2479732,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "9962496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7435
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656295917.875, "dur": 0.256,
    "args": {
      "External id": 2479733,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11404288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7436
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656295933.485, "dur": 22.078,
    "args": {
      "External id": 2479734,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7437
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656296055.125, "dur": 354.492,
    "args": {
      "External id": 2479735,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 7438
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514656296085.509, "dur": 319.409,
    "args": {
      "External id": 2479736,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 7439, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1336754,
    "ts": 1514656296095.782, "dur": 303.941,
    "args": {
      "External id": 2479737,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 7440
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514656296431.475, "dur": 2.102,
    "args": {
      "External id": 2479738,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 7441, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.18)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656296518.484, "dur": 16923.858,
    "args": {
      "External id": 2479739,"Record function id": 0, "Ev Idx": 7442
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656296638.514, "dur": 6.269,
    "args": {
      "External id": 2479740,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 7443
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656296648.808, "dur": 1.518,
    "args": {
      "External id": 2479741,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7444
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656296652.098, "dur": 2.599,
    "args": {
      "External id": 2479742,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7445
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656296656.534, "dur": 1.154,
    "args": {
      "External id": 2479743,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7446
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656296659.152, "dur": 0.854,
    "args": {
      "External id": 2479744,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7447
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656296661.161, "dur": 0.894,
    "args": {
      "External id": 2479745,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7448
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656296665.087, "dur": 0.841,
    "args": {
      "External id": 2479746,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7449
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656296667.723, "dur": 1.738,
    "args": {
      "External id": 2479747,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7450
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656296670.749, "dur": 0.797,
    "args": {
      "External id": 2479748,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7451
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656296672.831, "dur": 0.718,
    "args": {
      "External id": 2479749,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7452
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656296693.031, "dur": 16711.924,
    "args": {
      "External id": 2479750,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7453
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656296708.950, "dur": 16689.106,
    "args": {
      "External id": 2479751,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7454
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656296731.035, "dur": 13.842,
    "args": {
      "External id": 2479752,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7455
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514656296747.787, "dur": 16618.639,
    "args": {
      "External id": 2479753,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 7456
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656296750.140, "dur": 16615.802,
    "args": {
      "External id": 2479754,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 7457
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656296755.255, "dur": 5.795,
    "args": {
      "External id": 2479755,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7458
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656296762.618, "dur": 16600.394,
    "args": {
      "External id": 2479756,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 7459
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656313583.804, "dur": 36.910,
    "args": {
      "External id": 2479757,"Sequence number": 24740457, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 7460
    }
  },
  {
    "ph": "s", "id": 191, "pid": 1336754, "tid": 1336754, "ts": 1514656313583.804,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336754, "tid": 1336754,
    "ts": 1514656313608.661, "dur": 7.080,
    "args": {
      "External id": 2479758,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 7461
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656313611.517, "dur": 3.877,
    "args": {
      "External id": 2479759,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 7462
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336754, "tid": 1336754,
    "ts": 1514656313683.731, "dur": 73.038,
    "args": {
      "External id": 2479760,"Record function id": 0, "Ev Idx": 7463
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656313758.316, "dur": 1044.608,
    "args": {
      "External id": 2479761,"Record function id": 0, "Ev Idx": 7464
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656313798.110, "dur": 992.282,
    "args": {
      "External id": 2479762,"Sequence number": 24740458, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 7465
    }
  },
  {
    "ph": "s", "id": 190, "pid": 1336754, "tid": 1336754, "ts": 1514656313798.110,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514656313868.680, "dur": 42.037,
    "args": {
      "External id": 2479763,"kernel_hash": "caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/ao/caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7466
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656313924.759, "dur": 129.858,
    "args": {
      "External id": 2479764,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7467
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656314068.495, "dur": 44.022,
    "args": {
      "External id": 2479765,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7468
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656314119.436, "dur": 31.729,
    "args": {
      "External id": 2479766,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7469
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656314177.802, "dur": 25.006,
    "args": {
      "External id": 2479767,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7470
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656314219.344, "dur": 16.705,
    "args": {
      "External id": 2479768,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7471
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336754, "tid": 1336754,
    "ts": 1514656314252.529, "dur": 126.098,
    "args": {
      "External id": 2479769,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 7472
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656314300.694, "dur": 11.332,
    "args": {
      "External id": 2479770,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 7473
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656314305.564, "dur": 5.715,
    "args": {
      "External id": 2479771,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7474
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656314314.734, "dur": 5.997,
    "args": {
      "External id": 2479772,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7475
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656314322.019, "dur": 2.725,
    "args": {
      "External id": 2479773,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7476
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656314327.183, "dur": 4.266,
    "args": {
      "External id": 2479774,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7477
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656314388.809, "dur": 45.475,
    "args": {
      "External id": 2479775,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7478
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336754, "tid": 1336754,
    "ts": 1514656314462.606, "dur": 26.214,
    "args": {
      "External id": 2479776,"kernel_hash": "cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/wx/cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7479
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656314497.529, "dur": 57.067,
    "args": {
      "External id": 2479777,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7480
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656314563.867, "dur": 39.105,
    "args": {
      "External id": 2479778,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7481
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514656314626.659, "dur": 24.629,
    "args": {
      "External id": 2479779,"kernel_hash": "c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2f/c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 7482
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656314656.837, "dur": 34.809,
    "args": {
      "External id": 2479780,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 7483
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656314708.601, "dur": 19.152,
    "args": {
      "External id": 2479781,"kernel_hash": "cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/gi/cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 7484
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.18)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656314862.706, "dur": 71.462,
    "args": {
      "External id": 2479782,"Record function id": 0, "Ev Idx": 7485
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656315049.502, "dur": 48.908,
    "args": {
      "External id": 2479783,"Record function id": 0, "Ev Idx": 7486
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.19)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656315109.179, "dur": 18256.390,
    "args": {
      "External id": 2479784,"Record function id": 0, "Ev Idx": 7487
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.19)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656315117.162, "dur": 939.496,
    "args": {
      "External id": 2479785,"Record function id": 0, "Ev Idx": 7488
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656315199.946, "dur": 8.762,
    "args": {
      "External id": 2479786,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7489
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514656315222.282, "dur": 34.134,
    "args": {
      "External id": 2479787,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7490
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656315227.875, "dur": 2.400,
    "args": {
      "External id": 2479788,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7491
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656315234.163, "dur": 0.334,
    "args": {
      "External id": 2479789,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7492
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656315235.845, "dur": 0.323,
    "args": {
      "External id": 2479790,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7493
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656315237.517, "dur": 0.390,
    "args": {
      "External id": 2479791,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7494
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656315241.190, "dur": 0.321,
    "args": {
      "External id": 2479792,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7495
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656315242.953, "dur": 0.194,
    "args": {
      "External id": 2479793,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7496
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656315244.419, "dur": 2.781,
    "args": {
      "External id": 2479794,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7497
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656315248.526, "dur": 0.195,
    "args": {
      "External id": 2479795,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7498
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656315250.076, "dur": 0.158,
    "args": {
      "External id": 2479796,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7499
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656315267.528, "dur": 40.292,
    "args": {
      "External id": 2479797,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7500
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1336754,
    "ts": 1514656315340.304, "dur": 105.387,
    "args": {
      "External id": 2479798,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "1", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 7501
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656315350.773, "dur": 3.857,
    "args": {
      "External id": 2479799,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7502
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1336754,
    "ts": 1514656315359.535, "dur": 9.609,
    "args": {
      "External id": 2479800,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7503
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656315363.710, "dur": 5.020,
    "args": {
      "External id": 2479801,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "12846080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 7504
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656315367.015, "dur": 0.471,
    "args": {
      "External id": 2479802,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7505
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514656315376.368, "dur": 27.396,
    "args": {
      "External id": 2479803,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7506
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656315378.083, "dur": 2.277,
    "args": {
      "External id": 2479804,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7507
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656315381.892, "dur": 0.375,
    "args": {
      "External id": 2479805,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6423296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7508
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656315383.332, "dur": 0.304,
    "args": {
      "External id": 2479806,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6947584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7509
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656315386.507, "dur": 1.441,
    "args": {
      "External id": 2479807,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7471872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7510
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656315389.077, "dur": 0.393,
    "args": {
      "External id": 2479808,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7996160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7511
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656315390.889, "dur": 0.585,
    "args": {
      "External id": 2479809,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8520448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7512
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656315394.076, "dur": 0.423,
    "args": {
      "External id": 2479810,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "8520704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7513
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656315395.777, "dur": 0.285,
    "args": {
      "External id": 2479811,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "9962496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7514
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656315397.547, "dur": 1.847,
    "args": {
      "External id": 2479812,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11404288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7515
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656315415.196, "dur": 23.080,
    "args": {
      "External id": 2479813,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7516
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656315497.774, "dur": 428.485,
    "args": {
      "External id": 2479814,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 7517
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514656315545.081, "dur": 376.159,
    "args": {
      "External id": 2479815,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 7518, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1336754,
    "ts": 1514656315556.638, "dur": 357.327,
    "args": {
      "External id": 2479816,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 7519
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514656315950.634, "dur": 2.319,
    "args": {
      "External id": 2479817,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 7520, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.19)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656316079.187, "dur": 17092.016,
    "args": {
      "External id": 2479818,"Record function id": 0, "Ev Idx": 7521
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656316182.175, "dur": 6.074,
    "args": {
      "External id": 2479819,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 7522
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656316203.658, "dur": 1.429,
    "args": {
      "External id": 2479820,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7523
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656316206.723, "dur": 3.652,
    "args": {
      "External id": 2479821,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7524
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656316213.295, "dur": 0.757,
    "args": {
      "External id": 2479822,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7525
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656316215.553, "dur": 0.733,
    "args": {
      "External id": 2479823,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7526
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656316217.412, "dur": 1.228,
    "args": {
      "External id": 2479824,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7527
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656316220.460, "dur": 0.701,
    "args": {
      "External id": 2479825,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7528
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656316223.999, "dur": 1.614,
    "args": {
      "External id": 2479826,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7529
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656316227.333, "dur": 0.726,
    "args": {
      "External id": 2479827,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7530
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656316229.551, "dur": 0.523,
    "args": {
      "External id": 2479828,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7531
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656316250.428, "dur": 16881.257,
    "args": {
      "External id": 2479829,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7532
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656316266.437, "dur": 16858.042,
    "args": {
      "External id": 2479830,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7533
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656316284.235, "dur": 14.661,
    "args": {
      "External id": 2479831,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7534
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514656316303.983, "dur": 16787.616,
    "args": {
      "External id": 2479832,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 7535
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656316306.389, "dur": 16784.669,
    "args": {
      "External id": 2479833,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 7536
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656316311.936, "dur": 5.217,
    "args": {
      "External id": 2479834,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7537
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656316318.608, "dur": 16769.503,
    "args": {
      "External id": 2479835,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 7538
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656333304.021, "dur": 36.736,
    "args": {
      "External id": 2479836,"Sequence number": 24740459, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 7539
    }
  },
  {
    "ph": "s", "id": 189, "pid": 1336754, "tid": 1336754, "ts": 1514656333304.021,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336754, "tid": 1336754,
    "ts": 1514656333328.608, "dur": 6.965,
    "args": {
      "External id": 2479837,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 7540
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656333331.507, "dur": 3.786,
    "args": {
      "External id": 2479838,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 7541
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336754, "tid": 1336754,
    "ts": 1514656333403.912, "dur": 76.120,
    "args": {
      "External id": 2479839,"Record function id": 0, "Ev Idx": 7542
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656333481.398, "dur": 1074.078,
    "args": {
      "External id": 2479840,"Record function id": 0, "Ev Idx": 7543
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656333516.737, "dur": 1024.693,
    "args": {
      "External id": 2479841,"Sequence number": 24740460, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 7544
    }
  },
  {
    "ph": "s", "id": 188, "pid": 1336754, "tid": 1336754, "ts": 1514656333516.737,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514656333598.286, "dur": 42.297,
    "args": {
      "External id": 2479842,"kernel_hash": "caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/ao/caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7545
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656333653.486, "dur": 103.217,
    "args": {
      "External id": 2479843,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7546
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656333766.883, "dur": 41.389,
    "args": {
      "External id": 2479844,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7547
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656333816.517, "dur": 32.177,
    "args": {
      "External id": 2479845,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7548
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656333871.353, "dur": 24.414,
    "args": {
      "External id": 2479846,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7549
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656333913.051, "dur": 15.076,
    "args": {
      "External id": 2479847,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7550
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336754, "tid": 1336754,
    "ts": 1514656333945.331, "dur": 167.064,
    "args": {
      "External id": 2479848,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 7551
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656334030.335, "dur": 13.170,
    "args": {
      "External id": 2479849,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 7552
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656334035.732, "dur": 6.781,
    "args": {
      "External id": 2479850,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7553
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656334046.135, "dur": 6.383,
    "args": {
      "External id": 2479851,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7554
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656334053.776, "dur": 1.199,
    "args": {
      "External id": 2479852,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7555
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656334057.251, "dur": 3.545,
    "args": {
      "External id": 2479853,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7556
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656334123.190, "dur": 51.080,
    "args": {
      "External id": 2479854,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7557
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336754, "tid": 1336754,
    "ts": 1514656334204.803, "dur": 28.526,
    "args": {
      "External id": 2479855,"kernel_hash": "cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/wx/cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7558
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656334242.744, "dur": 42.582,
    "args": {
      "External id": 2479856,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7559
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656334293.752, "dur": 35.819,
    "args": {
      "External id": 2479857,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7560
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514656334350.517, "dur": 25.912,
    "args": {
      "External id": 2479858,"kernel_hash": "c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2f/c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 7561
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656334381.677, "dur": 35.068,
    "args": {
      "External id": 2479859,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 7562
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656334436.333, "dur": 16.909,
    "args": {
      "External id": 2479860,"kernel_hash": "cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/gi/cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 7563
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.19)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656334619.445, "dur": 72.755,
    "args": {
      "External id": 2479861,"Record function id": 0, "Ev Idx": 7564
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656334761.153, "dur": 44.858,
    "args": {
      "External id": 2479862,"Record function id": 0, "Ev Idx": 7565
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.20)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656334814.895, "dur": 18342.039,
    "args": {
      "External id": 2479863,"Record function id": 0, "Ev Idx": 7566
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.20)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656334823.634, "dur": 905.583,
    "args": {
      "External id": 2479864,"Record function id": 0, "Ev Idx": 7567
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656334898.811, "dur": 7.541,
    "args": {
      "External id": 2479865,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7568
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514656334919.558, "dur": 34.420,
    "args": {
      "External id": 2479866,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7569
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656334925.021, "dur": 2.098,
    "args": {
      "External id": 2479867,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7570
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656334931.112, "dur": 0.400,
    "args": {
      "External id": 2479868,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7571
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656334932.745, "dur": 0.498,
    "args": {
      "External id": 2479869,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7572
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656334934.480, "dur": 0.458,
    "args": {
      "External id": 2479870,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7573
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656334938.236, "dur": 0.448,
    "args": {
      "External id": 2479871,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7574
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656334939.974, "dur": 0.487,
    "args": {
      "External id": 2479872,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7575
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656334941.814, "dur": 2.228,
    "args": {
      "External id": 2479873,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7576
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656334945.663, "dur": 0.331,
    "args": {
      "External id": 2479874,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7577
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656334947.283, "dur": 0.230,
    "args": {
      "External id": 2479875,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7578
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656334964.406, "dur": 73.615,
    "args": {
      "External id": 2479876,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7579
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1336754,
    "ts": 1514656335075.188, "dur": 115.363,
    "args": {
      "External id": 2479877,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "1", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 7580
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656335086.140, "dur": 4.925,
    "args": {
      "External id": 2479878,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7581
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1336754,
    "ts": 1514656335096.060, "dur": 11.005,
    "args": {
      "External id": 2479879,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7582
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656335100.424, "dur": 6.227,
    "args": {
      "External id": 2479880,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "12846080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 7583
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656335104.062, "dur": 0.864,
    "args": {
      "External id": 2479881,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7584
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514656335114.616, "dur": 31.210,
    "args": {
      "External id": 2479882,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7585
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656335117.098, "dur": 2.120,
    "args": {
      "External id": 2479883,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7586
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656335120.502, "dur": 0.332,
    "args": {
      "External id": 2479884,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6423296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7587
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656335125.246, "dur": 0.415,
    "args": {
      "External id": 2479885,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6947584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7588
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656335128.632, "dur": 1.861,
    "args": {
      "External id": 2479886,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7471872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7589
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656335131.934, "dur": 0.463,
    "args": {
      "External id": 2479887,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7996160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7590
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656335133.479, "dur": 1.592,
    "args": {
      "External id": 2479888,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8520448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7591
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656335136.249, "dur": 0.353,
    "args": {
      "External id": 2479889,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "8520704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7592
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656335138.171, "dur": 0.361,
    "args": {
      "External id": 2479890,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "9962496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7593
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656335141.250, "dur": 0.303,
    "args": {
      "External id": 2479891,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11404288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7594
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656335157.164, "dur": 25.812,
    "args": {
      "External id": 2479892,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7595
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656335243.879, "dur": 388.341,
    "args": {
      "External id": 2479893,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 7596
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514656335274.120, "dur": 352.521,
    "args": {
      "External id": 2479894,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 7597, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1336754,
    "ts": 1514656335284.024, "dur": 336.852,
    "args": {
      "External id": 2479895,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 7598
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514656335657.752, "dur": 2.490,
    "args": {
      "External id": 2479896,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 7599, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.20)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656335751.261, "dur": 17185.801,
    "args": {
      "External id": 2479897,"Record function id": 0, "Ev Idx": 7600
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656335847.760, "dur": 6.338,
    "args": {
      "External id": 2479898,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 7601
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656335857.450, "dur": 1.435,
    "args": {
      "External id": 2479899,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7602
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656335860.448, "dur": 2.226,
    "args": {
      "External id": 2479900,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7603
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656335864.252, "dur": 0.770,
    "args": {
      "External id": 2479901,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7604
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656335866.259, "dur": 0.775,
    "args": {
      "External id": 2479902,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7605
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656335868.229, "dur": 1.189,
    "args": {
      "External id": 2479903,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7606
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656335872.554, "dur": 0.788,
    "args": {
      "External id": 2479904,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7607
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656335874.788, "dur": 2.053,
    "args": {
      "External id": 2479905,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7608
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656335878.098, "dur": 0.803,
    "args": {
      "External id": 2479906,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7609
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656335880.252, "dur": 0.613,
    "args": {
      "External id": 2479907,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7610
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656335900.741, "dur": 16999.852,
    "args": {
      "External id": 2479908,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7611
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656335915.840, "dur": 16978.281,
    "args": {
      "External id": 2479909,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7612
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656335936.560, "dur": 15.091,
    "args": {
      "External id": 2479910,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7613
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514656335954.630, "dur": 16908.658,
    "args": {
      "External id": 2479911,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 7614
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656335957.023, "dur": 16905.725,
    "args": {
      "External id": 2479912,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 7615
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656335962.467, "dur": 5.356,
    "args": {
      "External id": 2479913,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7616
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656335969.404, "dur": 16890.536,
    "args": {
      "External id": 2479914,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 7617
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656353093.457, "dur": 38.237,
    "args": {
      "External id": 2479915,"Sequence number": 24740461, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 7618
    }
  },
  {
    "ph": "s", "id": 187, "pid": 1336754, "tid": 1336754, "ts": 1514656353093.457,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336754, "tid": 1336754,
    "ts": 1514656353118.969, "dur": 7.891,
    "args": {
      "External id": 2479916,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 7619
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656353122.283, "dur": 4.256,
    "args": {
      "External id": 2479917,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 7620
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336754, "tid": 1336754,
    "ts": 1514656353194.640, "dur": 72.724,
    "args": {
      "External id": 2479918,"Record function id": 0, "Ev Idx": 7621
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656353268.956, "dur": 1064.362,
    "args": {
      "External id": 2479919,"Record function id": 0, "Ev Idx": 7622
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656353308.199, "dur": 1012.184,
    "args": {
      "External id": 2479920,"Sequence number": 24740462, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 7623
    }
  },
  {
    "ph": "s", "id": 186, "pid": 1336754, "tid": 1336754, "ts": 1514656353308.199,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514656353370.163, "dur": 41.341,
    "args": {
      "External id": 2479921,"kernel_hash": "caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/ao/caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7624
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656353423.276, "dur": 120.743,
    "args": {
      "External id": 2479922,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7625
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656353558.454, "dur": 44.528,
    "args": {
      "External id": 2479923,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7626
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656353609.523, "dur": 32.576,
    "args": {
      "External id": 2479924,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7627
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656353669.529, "dur": 27.184,
    "args": {
      "External id": 2479925,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7628
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656353711.698, "dur": 15.508,
    "args": {
      "External id": 2479926,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7629
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336754, "tid": 1336754,
    "ts": 1514656353744.793, "dur": 124.146,
    "args": {
      "External id": 2479927,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 7630
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656353791.739, "dur": 10.789,
    "args": {
      "External id": 2479928,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 7631
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656353796.449, "dur": 5.212,
    "args": {
      "External id": 2479929,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7632
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656353805.171, "dur": 5.592,
    "args": {
      "External id": 2479930,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7633
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656353812.175, "dur": 1.063,
    "args": {
      "External id": 2479931,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7634
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656353817.357, "dur": 3.904,
    "args": {
      "External id": 2479932,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7635
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656353878.653, "dur": 46.795,
    "args": {
      "External id": 2479933,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7636
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336754, "tid": 1336754,
    "ts": 1514656353953.555, "dur": 66.006,
    "args": {
      "External id": 2479934,"kernel_hash": "cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/wx/cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7637
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656354033.890, "dur": 48.194,
    "args": {
      "External id": 2479935,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7638
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656354088.603, "dur": 36.132,
    "args": {
      "External id": 2479936,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7639
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514656354148.593, "dur": 29.386,
    "args": {
      "External id": 2479937,"kernel_hash": "c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2f/c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 7640
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656354183.597, "dur": 35.184,
    "args": {
      "External id": 2479938,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 7641
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656354236.527, "dur": 18.640,
    "args": {
      "External id": 2479939,"kernel_hash": "cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/gi/cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 7642
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.20)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656354394.840, "dur": 71.650,
    "args": {
      "External id": 2479940,"Record function id": 0, "Ev Idx": 7643
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656354556.080, "dur": 47.627,
    "args": {
      "External id": 2479941,"Record function id": 0, "Ev Idx": 7644
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.21)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656354613.473, "dur": 18198.201,
    "args": {
      "External id": 2479942,"Record function id": 0, "Ev Idx": 7645
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.21)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656354623.595, "dur": 871.166,
    "args": {
      "External id": 2479943,"Record function id": 0, "Ev Idx": 7646
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656354704.810, "dur": 8.861,
    "args": {
      "External id": 2479944,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7647
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514656354727.956, "dur": 34.320,
    "args": {
      "External id": 2479945,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7648
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656354732.980, "dur": 2.383,
    "args": {
      "External id": 2479946,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7649
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656354739.268, "dur": 0.202,
    "args": {
      "External id": 2479947,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7650
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656354741.138, "dur": 0.442,
    "args": {
      "External id": 2479948,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7651
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656354742.938, "dur": 0.613,
    "args": {
      "External id": 2479949,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7652
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656354746.349, "dur": 0.403,
    "args": {
      "External id": 2479950,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7653
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656354748.090, "dur": 0.321,
    "args": {
      "External id": 2479951,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7654
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656354749.780, "dur": 3.123,
    "args": {
      "External id": 2479952,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7655
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656354754.373, "dur": 0.332,
    "args": {
      "External id": 2479953,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7656
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656354756.171, "dur": 0.227,
    "args": {
      "External id": 2479954,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7657
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656354772.723, "dur": 40.438,
    "args": {
      "External id": 2479955,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7658
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1336754,
    "ts": 1514656354844.487, "dur": 108.901,
    "args": {
      "External id": 2479956,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "1", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 7659
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656354855.570, "dur": 3.878,
    "args": {
      "External id": 2479957,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7660
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1336754,
    "ts": 1514656354864.642, "dur": 9.907,
    "args": {
      "External id": 2479958,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7661
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656354868.780, "dur": 5.376,
    "args": {
      "External id": 2479959,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "12846080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 7662
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656354872.290, "dur": 0.745,
    "args": {
      "External id": 2479960,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7663
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514656354881.279, "dur": 28.718,
    "args": {
      "External id": 2479961,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7664
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656354883.384, "dur": 2.124,
    "args": {
      "External id": 2479962,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7665
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656354886.912, "dur": 0.428,
    "args": {
      "External id": 2479963,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6423296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7666
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656354888.581, "dur": 0.449,
    "args": {
      "External id": 2479964,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6947584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7667
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656354892.146, "dur": 1.633,
    "args": {
      "External id": 2479965,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7471872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7668
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656354895.450, "dur": 0.296,
    "args": {
      "External id": 2479966,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7996160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7669
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656354896.958, "dur": 0.264,
    "args": {
      "External id": 2479967,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8520448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7670
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656354899.866, "dur": 0.349,
    "args": {
      "External id": 2479968,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "8520704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7671
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656354901.782, "dur": 0.419,
    "args": {
      "External id": 2479969,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "9962496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7672
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656354903.365, "dur": 1.923,
    "args": {
      "External id": 2479970,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11404288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7673
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656354921.934, "dur": 23.683,
    "args": {
      "External id": 2479971,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7674
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656355049.921, "dur": 355.183,
    "args": {
      "External id": 2479972,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 7675
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514656355082.822, "dur": 317.536,
    "args": {
      "External id": 2479973,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 7676, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1336754,
    "ts": 1514656355093.280, "dur": 301.804,
    "args": {
      "External id": 2479974,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 7677
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514656355426.162, "dur": 2.367,
    "args": {
      "External id": 2479975,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 7678, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.21)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656355514.318, "dur": 17113.490,
    "args": {
      "External id": 2479976,"Record function id": 0, "Ev Idx": 7679
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656355632.219, "dur": 6.698,
    "args": {
      "External id": 2479977,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 7680
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656355642.540, "dur": 1.360,
    "args": {
      "External id": 2479978,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7681
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656355645.595, "dur": 2.310,
    "args": {
      "External id": 2479979,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7682
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656355649.774, "dur": 0.900,
    "args": {
      "External id": 2479980,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7683
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656355651.906, "dur": 0.931,
    "args": {
      "External id": 2479981,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7684
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656355654.123, "dur": 0.968,
    "args": {
      "External id": 2479982,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7685
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656355658.283, "dur": 0.843,
    "args": {
      "External id": 2479983,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7686
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656355660.527, "dur": 1.688,
    "args": {
      "External id": 2479984,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7687
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656355663.647, "dur": 0.790,
    "args": {
      "External id": 2479985,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7688
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656355665.911, "dur": 0.866,
    "args": {
      "External id": 2479986,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7689
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656355686.461, "dur": 16903.449,
    "args": {
      "External id": 2479987,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7690
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656355702.787, "dur": 16879.939,
    "args": {
      "External id": 2479988,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7691
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656355722.390, "dur": 15.082,
    "args": {
      "External id": 2479989,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7692
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514656355740.395, "dur": 16810.443,
    "args": {
      "External id": 2479990,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 7693
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656355742.712, "dur": 16807.605,
    "args": {
      "External id": 2479991,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 7694
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656355747.947, "dur": 5.191,
    "args": {
      "External id": 2479992,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7695
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656355754.647, "dur": 16792.634,
    "args": {
      "External id": 2479993,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 7696
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656372753.903, "dur": 34.296,
    "args": {
      "External id": 2479994,"Sequence number": 24740463, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 7697
    }
  },
  {
    "ph": "s", "id": 185, "pid": 1336754, "tid": 1336754, "ts": 1514656372753.903,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336754, "tid": 1336754,
    "ts": 1514656372776.066, "dur": 7.145,
    "args": {
      "External id": 2479995,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 7698
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656372779.012, "dur": 4.010,
    "args": {
      "External id": 2479996,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 7699
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336754, "tid": 1336754,
    "ts": 1514656372848.051, "dur": 74.035,
    "args": {
      "External id": 2479997,"Record function id": 0, "Ev Idx": 7700
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656372923.716, "dur": 1040.693,
    "args": {
      "External id": 2479998,"Record function id": 0, "Ev Idx": 7701
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656372963.049, "dur": 988.194,
    "args": {
      "External id": 2479999,"Sequence number": 24740464, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 7702
    }
  },
  {
    "ph": "s", "id": 184, "pid": 1336754, "tid": 1336754, "ts": 1514656372963.049,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514656373057.286, "dur": 42.097,
    "args": {
      "External id": 2480000,"kernel_hash": "caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/ao/caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7703
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656373112.583, "dur": 102.887,
    "args": {
      "External id": 2480001,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7704
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656373225.018, "dur": 40.890,
    "args": {
      "External id": 2480002,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7705
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656373274.168, "dur": 31.961,
    "args": {
      "External id": 2480003,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7706
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656373330.191, "dur": 24.402,
    "args": {
      "External id": 2480004,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7707
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656373371.012, "dur": 14.605,
    "args": {
      "External id": 2480005,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7708
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336754, "tid": 1336754,
    "ts": 1514656373403.774, "dur": 122.434,
    "args": {
      "External id": 2480006,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 7709
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656373451.803, "dur": 10.780,
    "args": {
      "External id": 2480007,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 7710
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656373456.771, "dur": 5.112,
    "args": {
      "External id": 2480008,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7711
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656373465.210, "dur": 5.276,
    "args": {
      "External id": 2480009,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7712
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656373471.893, "dur": 1.210,
    "args": {
      "External id": 2480010,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7713
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656373475.303, "dur": 3.439,
    "args": {
      "External id": 2480011,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7714
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656373553.260, "dur": 52.329,
    "args": {
      "External id": 2480012,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7715
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336754, "tid": 1336754,
    "ts": 1514656373635.961, "dur": 29.918,
    "args": {
      "External id": 2480013,"kernel_hash": "cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/wx/cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7716
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656373674.936, "dur": 40.994,
    "args": {
      "External id": 2480014,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7717
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656373724.160, "dur": 35.279,
    "args": {
      "External id": 2480015,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7718
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514656373781.644, "dur": 26.879,
    "args": {
      "External id": 2480016,"kernel_hash": "c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2f/c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 7719
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656373813.917, "dur": 34.306,
    "args": {
      "External id": 2480017,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 7720
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656373866.674, "dur": 18.117,
    "args": {
      "External id": 2480018,"kernel_hash": "cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/gi/cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 7721
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.21)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656374060.567, "dur": 75.001,
    "args": {
      "External id": 2480019,"Record function id": 0, "Ev Idx": 7722
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656374206.834, "dur": 46.061,
    "args": {
      "External id": 2480020,"Record function id": 0, "Ev Idx": 7723
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.22)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656374262.562, "dur": 18152.811,
    "args": {
      "External id": 2480021,"Record function id": 0, "Ev Idx": 7724
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.22)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656374272.106, "dur": 903.967,
    "args": {
      "External id": 2480022,"Record function id": 0, "Ev Idx": 7725
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656374351.534, "dur": 8.419,
    "args": {
      "External id": 2480023,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7726
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514656374373.106, "dur": 34.148,
    "args": {
      "External id": 2480024,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7727
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656374378.397, "dur": 2.233,
    "args": {
      "External id": 2480025,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7728
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656374384.629, "dur": 0.485,
    "args": {
      "External id": 2480026,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7729
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656374386.435, "dur": 0.420,
    "args": {
      "External id": 2480027,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7730
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656374388.424, "dur": 0.457,
    "args": {
      "External id": 2480028,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7731
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656374391.761, "dur": 0.424,
    "args": {
      "External id": 2480029,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7732
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656374393.419, "dur": 0.374,
    "args": {
      "External id": 2480030,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7733
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656374394.881, "dur": 2.999,
    "args": {
      "External id": 2480031,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7734
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656374399.110, "dur": 0.242,
    "args": {
      "External id": 2480032,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7735
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656374400.444, "dur": 0.301,
    "args": {
      "External id": 2480033,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7736
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656374417.728, "dur": 39.561,
    "args": {
      "External id": 2480034,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7737
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1336754,
    "ts": 1514656374488.141, "dur": 132.457,
    "args": {
      "External id": 2480035,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "1", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 7738
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656374498.613, "dur": 4.192,
    "args": {
      "External id": 2480036,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7739
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1336754,
    "ts": 1514656374507.832, "dur": 10.049,
    "args": {
      "External id": 2480037,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7740
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656374512.041, "dur": 5.451,
    "args": {
      "External id": 2480038,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "12846080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 7741
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656374515.732, "dur": 0.531,
    "args": {
      "External id": 2480039,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7742
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514656374524.673, "dur": 50.277,
    "args": {
      "External id": 2480040,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7743
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656374542.669, "dur": 2.435,
    "args": {
      "External id": 2480041,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7744
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656374547.547, "dur": 0.435,
    "args": {
      "External id": 2480042,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6423296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7745
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656374549.271, "dur": 0.325,
    "args": {
      "External id": 2480043,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6947584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7746
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656374552.637, "dur": 1.623,
    "args": {
      "External id": 2480044,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7471872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7747
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656374555.889, "dur": 0.490,
    "args": {
      "External id": 2480045,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7996160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7748
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656374557.531, "dur": 0.352,
    "args": {
      "External id": 2480046,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8520448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7749
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656374560.532, "dur": 0.336,
    "args": {
      "External id": 2480047,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "8520704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7750
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656374562.400, "dur": 0.263,
    "args": {
      "External id": 2480048,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "9962496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7751
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656374567.630, "dur": 1.841,
    "args": {
      "External id": 2480049,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11404288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7752
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656374587.037, "dur": 26.195,
    "args": {
      "External id": 2480050,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7753
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656374672.954, "dur": 406.885,
    "args": {
      "External id": 2480051,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 7754
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514656374704.099, "dur": 370.000,
    "args": {
      "External id": 2480052,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 7755, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1336754,
    "ts": 1514656374714.042, "dur": 354.177,
    "args": {
      "External id": 2480053,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 7756
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514656375105.284, "dur": 2.548,
    "args": {
      "External id": 2480054,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 7757, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.22)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656375197.712, "dur": 17031.185,
    "args": {
      "External id": 2480055,"Record function id": 0, "Ev Idx": 7758
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656375295.088, "dur": 6.313,
    "args": {
      "External id": 2480056,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 7759
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656375304.700, "dur": 1.096,
    "args": {
      "External id": 2480057,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7760
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656375307.382, "dur": 2.269,
    "args": {
      "External id": 2480058,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7761
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656375311.113, "dur": 0.794,
    "args": {
      "External id": 2480059,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7762
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656375313.349, "dur": 0.862,
    "args": {
      "External id": 2480060,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7763
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656375315.295, "dur": 0.773,
    "args": {
      "External id": 2480061,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7764
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656375319.318, "dur": 0.955,
    "args": {
      "External id": 2480062,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7765
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656375321.508, "dur": 1.877,
    "args": {
      "External id": 2480063,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7766
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656375324.844, "dur": 0.618,
    "args": {
      "External id": 2480064,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7767
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656375326.949, "dur": 0.854,
    "args": {
      "External id": 2480065,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7768
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656375347.738, "dur": 16841.613,
    "args": {
      "External id": 2480066,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7769
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656375368.954, "dur": 16813.176,
    "args": {
      "External id": 2480067,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7770
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656375387.919, "dur": 13.417,
    "args": {
      "External id": 2480068,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7771
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514656375404.235, "dur": 16746.317,
    "args": {
      "External id": 2480069,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 7772
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656375406.767, "dur": 16743.159,
    "args": {
      "External id": 2480070,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 7773
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656375412.052, "dur": 6.029,
    "args": {
      "External id": 2480071,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7774
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656375419.591, "dur": 16727.279,
    "args": {
      "External id": 2480072,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 7775
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656392356.837, "dur": 34.479,
    "args": {
      "External id": 2480073,"Sequence number": 24740465, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 7776
    }
  },
  {
    "ph": "s", "id": 183, "pid": 1336754, "tid": 1336754, "ts": 1514656392356.837,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336754, "tid": 1336754,
    "ts": 1514656392379.139, "dur": 7.258,
    "args": {
      "External id": 2480074,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 7777
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656392382.492, "dur": 3.732,
    "args": {
      "External id": 2480075,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 7778
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336754, "tid": 1336754,
    "ts": 1514656392452.721, "dur": 86.031,
    "args": {
      "External id": 2480076,"Record function id": 0, "Ev Idx": 7779
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656392541.879, "dur": 1070.586,
    "args": {
      "External id": 2480077,"Record function id": 0, "Ev Idx": 7780
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656392582.529, "dur": 1016.491,
    "args": {
      "External id": 2480078,"Sequence number": 24740466, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 7781
    }
  },
  {
    "ph": "s", "id": 182, "pid": 1336754, "tid": 1336754, "ts": 1514656392582.529,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514656392646.093, "dur": 43.254,
    "args": {
      "External id": 2480079,"kernel_hash": "caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/ao/caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7782
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656392701.990, "dur": 125.516,
    "args": {
      "External id": 2480080,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7783
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656392837.482, "dur": 38.336,
    "args": {
      "External id": 2480081,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7784
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656392883.994, "dur": 31.867,
    "args": {
      "External id": 2480082,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7785
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656392939.726, "dur": 24.199,
    "args": {
      "External id": 2480083,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7786
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656392980.227, "dur": 54.363,
    "args": {
      "External id": 2480084,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7787
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336754, "tid": 1336754,
    "ts": 1514656393054.176, "dur": 123.432,
    "args": {
      "External id": 2480085,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 7788
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656393101.661, "dur": 11.712,
    "args": {
      "External id": 2480086,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 7789
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656393106.380, "dur": 6.159,
    "args": {
      "External id": 2480087,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7790
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656393115.960, "dur": 5.790,
    "args": {
      "External id": 2480088,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7791
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656393123.056, "dur": 1.097,
    "args": {
      "External id": 2480089,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7792
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656393126.564, "dur": 3.887,
    "args": {
      "External id": 2480090,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7793
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656393187.810, "dur": 51.132,
    "args": {
      "External id": 2480091,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7794
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336754, "tid": 1336754,
    "ts": 1514656393268.400, "dur": 28.447,
    "args": {
      "External id": 2480092,"kernel_hash": "cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/wx/cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7795
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656393305.166, "dur": 42.327,
    "args": {
      "External id": 2480093,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7796
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656393355.488, "dur": 35.825,
    "args": {
      "External id": 2480094,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7797
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514656393413.546, "dur": 28.733,
    "args": {
      "External id": 2480095,"kernel_hash": "c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2f/c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 7798
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656393447.502, "dur": 34.263,
    "args": {
      "External id": 2480096,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 7799
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656393500.474, "dur": 17.287,
    "args": {
      "External id": 2480097,"kernel_hash": "cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/gi/cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 7800
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.22)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656393672.840, "dur": 70.462,
    "args": {
      "External id": 2480098,"Record function id": 0, "Ev Idx": 7801
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656393810.880, "dur": 45.275,
    "args": {
      "External id": 2480099,"Record function id": 0, "Ev Idx": 7802
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.23)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656393865.504, "dur": 18403.555,
    "args": {
      "External id": 2480100,"Record function id": 0, "Ev Idx": 7803
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.23)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656393876.621, "dur": 914.534,
    "args": {
      "External id": 2480101,"Record function id": 0, "Ev Idx": 7804
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656393954.045, "dur": 8.378,
    "args": {
      "External id": 2480102,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7805
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514656393975.518, "dur": 72.852,
    "args": {
      "External id": 2480103,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7806
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656393980.574, "dur": 38.622,
    "args": {
      "External id": 2480104,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7807
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656394024.664, "dur": 0.423,
    "args": {
      "External id": 2480105,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7808
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656394026.620, "dur": 0.668,
    "args": {
      "External id": 2480106,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7809
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656394028.555, "dur": 0.176,
    "args": {
      "External id": 2480107,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7810
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656394031.520, "dur": 0.312,
    "args": {
      "External id": 2480108,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7811
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656394033.342, "dur": 0.480,
    "args": {
      "External id": 2480109,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7812
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656394035.059, "dur": 3.201,
    "args": {
      "External id": 2480110,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7813
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656394039.853, "dur": 0.301,
    "args": {
      "External id": 2480111,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7814
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656394041.484, "dur": 0.247,
    "args": {
      "External id": 2480112,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7815
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656394060.302, "dur": 42.185,
    "args": {
      "External id": 2480113,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7816
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1336754,
    "ts": 1514656394137.885, "dur": 131.325,
    "args": {
      "External id": 2480114,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "1", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 7817
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656394148.947, "dur": 4.682,
    "args": {
      "External id": 2480115,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7818
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1336754,
    "ts": 1514656394158.655, "dur": 9.871,
    "args": {
      "External id": 2480116,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7819
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656394162.742, "dur": 5.386,
    "args": {
      "External id": 2480117,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "12846080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 7820
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656394166.324, "dur": 0.531,
    "args": {
      "External id": 2480118,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7821
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514656394175.270, "dur": 27.537,
    "args": {
      "External id": 2480119,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7822
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656394177.554, "dur": 1.993,
    "args": {
      "External id": 2480120,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7823
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656394180.893, "dur": 0.315,
    "args": {
      "External id": 2480121,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6423296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7824
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656394182.589, "dur": 0.331,
    "args": {
      "External id": 2480122,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6947584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7825
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656394186.316, "dur": 1.137,
    "args": {
      "External id": 2480123,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7471872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7826
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656394188.872, "dur": 0.142,
    "args": {
      "External id": 2480124,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7996160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7827
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656394190.220, "dur": 0.293,
    "args": {
      "External id": 2480125,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8520448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7828
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656394192.988, "dur": 0.358,
    "args": {
      "External id": 2480126,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "8520704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7829
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656394195.419, "dur": 0.174,
    "args": {
      "External id": 2480127,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "9962496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7830
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656394196.799, "dur": 1.769,
    "args": {
      "External id": 2480128,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11404288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7831
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656394237.705, "dur": 23.624,
    "args": {
      "External id": 2480129,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7832
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656394322.072, "dur": 376.375,
    "args": {
      "External id": 2480130,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 7833
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514656394351.698, "dur": 341.376,
    "args": {
      "External id": 2480131,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 7834, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1336754,
    "ts": 1514656394362.683, "dur": 324.385,
    "args": {
      "External id": 2480132,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 7835
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514656394721.408, "dur": 2.390,
    "args": {
      "External id": 2480133,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 7836, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.23)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656394814.124, "dur": 17264.740,
    "args": {
      "External id": 2480134,"Record function id": 0, "Ev Idx": 7837
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656394911.734, "dur": 6.262,
    "args": {
      "External id": 2480135,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 7838
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656394921.547, "dur": 1.216,
    "args": {
      "External id": 2480136,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7839
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656394924.704, "dur": 1.910,
    "args": {
      "External id": 2480137,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7840
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656394928.428, "dur": 0.950,
    "args": {
      "External id": 2480138,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7841
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656394930.715, "dur": 0.814,
    "args": {
      "External id": 2480139,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7842
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656394932.701, "dur": 0.688,
    "args": {
      "External id": 2480140,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7843
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656394936.518, "dur": 0.936,
    "args": {
      "External id": 2480141,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7844
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656394938.905, "dur": 1.726,
    "args": {
      "External id": 2480142,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7845
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656394942.053, "dur": 0.624,
    "args": {
      "External id": 2480143,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7846
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656394944.270, "dur": 0.742,
    "args": {
      "External id": 2480144,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7847
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656394964.336, "dur": 17075.872,
    "args": {
      "External id": 2480145,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7848
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656394979.702, "dur": 17053.210,
    "args": {
      "External id": 2480146,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7849
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656395036.637, "dur": 15.021,
    "args": {
      "External id": 2480147,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7850
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514656395054.762, "dur": 16923.014,
    "args": {
      "External id": 2480148,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 7851
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656395057.358, "dur": 16919.950,
    "args": {
      "External id": 2480149,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 7852
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656395062.423, "dur": 6.058,
    "args": {
      "External id": 2480150,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7853
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656395070.058, "dur": 16904.388,
    "args": {
      "External id": 2480151,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 7854
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656412207.344, "dur": 37.570,
    "args": {
      "External id": 2480152,"Sequence number": 24740467, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 7855
    }
  },
  {
    "ph": "s", "id": 181, "pid": 1336754, "tid": 1336754, "ts": 1514656412207.344,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336754, "tid": 1336754,
    "ts": 1514656412232.579, "dur": 7.467,
    "args": {
      "External id": 2480153,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 7856
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656412235.462, "dur": 4.387,
    "args": {
      "External id": 2480154,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 7857
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336754, "tid": 1336754,
    "ts": 1514656412305.896, "dur": 74.418,
    "args": {
      "External id": 2480155,"Record function id": 0, "Ev Idx": 7858
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656412382.134, "dur": 1061.596,
    "args": {
      "External id": 2480156,"Record function id": 0, "Ev Idx": 7859
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656412423.619, "dur": 1006.917,
    "args": {
      "External id": 2480157,"Sequence number": 24740468, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 7860
    }
  },
  {
    "ph": "s", "id": 180, "pid": 1336754, "tid": 1336754, "ts": 1514656412423.619,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514656412486.217, "dur": 38.080,
    "args": {
      "External id": 2480158,"kernel_hash": "caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/ao/caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7861
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656412553.179, "dur": 106.270,
    "args": {
      "External id": 2480159,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7862
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656412669.392, "dur": 39.727,
    "args": {
      "External id": 2480160,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7863
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656412717.569, "dur": 31.488,
    "args": {
      "External id": 2480161,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7864
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656412774.624, "dur": 27.943,
    "args": {
      "External id": 2480162,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7865
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656412821.686, "dur": 16.838,
    "args": {
      "External id": 2480163,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7866
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336754, "tid": 1336754,
    "ts": 1514656412855.958, "dur": 124.003,
    "args": {
      "External id": 2480164,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 7867
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656412903.724, "dur": 10.235,
    "args": {
      "External id": 2480165,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 7868
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656412908.000, "dur": 5.242,
    "args": {
      "External id": 2480166,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7869
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656412916.795, "dur": 5.006,
    "args": {
      "External id": 2480167,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7870
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656412923.201, "dur": 1.162,
    "args": {
      "External id": 2480168,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7871
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656412926.720, "dur": 4.099,
    "args": {
      "External id": 2480169,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7872
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656413027.386, "dur": 52.849,
    "args": {
      "External id": 2480170,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7873
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336754, "tid": 1336754,
    "ts": 1514656413110.702, "dur": 29.589,
    "args": {
      "External id": 2480171,"kernel_hash": "cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/wx/cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7874
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656413148.771, "dur": 43.110,
    "args": {
      "External id": 2480172,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7875
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656413200.308, "dur": 35.251,
    "args": {
      "External id": 2480173,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7876
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514656413257.572, "dur": 26.084,
    "args": {
      "External id": 2480174,"kernel_hash": "c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2f/c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 7877
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656413289.525, "dur": 34.894,
    "args": {
      "External id": 2480175,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 7878
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656413344.059, "dur": 17.808,
    "args": {
      "External id": 2480176,"kernel_hash": "cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/gi/cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 7879
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.23)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656413503.331, "dur": 89.679,
    "args": {
      "External id": 2480177,"Record function id": 0, "Ev Idx": 7880
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656413666.938, "dur": 48.530,
    "args": {
      "External id": 2480178,"Record function id": 0, "Ev Idx": 7881
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.24)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656413724.495, "dur": 18320.873,
    "args": {
      "External id": 2480179,"Record function id": 0, "Ev Idx": 7882
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.24)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656413732.278, "dur": 899.083,
    "args": {
      "External id": 2480180,"Record function id": 0, "Ev Idx": 7883
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656413813.721, "dur": 8.426,
    "args": {
      "External id": 2480181,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7884
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514656413834.600, "dur": 34.853,
    "args": {
      "External id": 2480182,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7885
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656413840.006, "dur": 2.419,
    "args": {
      "External id": 2480183,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7886
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656413846.416, "dur": 0.398,
    "args": {
      "External id": 2480184,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7887
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656413848.261, "dur": 0.308,
    "args": {
      "External id": 2480185,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7888
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656413849.949, "dur": 0.424,
    "args": {
      "External id": 2480186,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7889
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656413853.282, "dur": 0.432,
    "args": {
      "External id": 2480187,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7890
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656413854.962, "dur": 0.525,
    "args": {
      "External id": 2480188,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7891
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656413856.578, "dur": 3.594,
    "args": {
      "External id": 2480189,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7892
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656413861.385, "dur": 0.397,
    "args": {
      "External id": 2480190,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7893
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656413862.896, "dur": 0.348,
    "args": {
      "External id": 2480191,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7894
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656413880.397, "dur": 39.840,
    "args": {
      "External id": 2480192,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7895
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1336754,
    "ts": 1514656413952.774, "dur": 151.137,
    "args": {
      "External id": 2480193,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "1", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 7896
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656413962.927, "dur": 3.985,
    "args": {
      "External id": 2480194,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7897
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1336754,
    "ts": 1514656413971.632, "dur": 46.351,
    "args": {
      "External id": 2480195,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7898
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656413976.108, "dur": 41.067,
    "args": {
      "External id": 2480196,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "12846080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 7899
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656413979.778, "dur": 0.517,
    "args": {
      "External id": 2480197,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7900
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514656414028.638, "dur": 28.869,
    "args": {
      "External id": 2480198,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7901
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656414031.426, "dur": 2.304,
    "args": {
      "External id": 2480199,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7902
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656414035.439, "dur": 0.394,
    "args": {
      "External id": 2480200,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6423296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7903
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656414037.108, "dur": 1.618,
    "args": {
      "External id": 2480201,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6947584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7904
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656414041.803, "dur": 1.593,
    "args": {
      "External id": 2480202,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7471872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7905
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656414044.379, "dur": 0.308,
    "args": {
      "External id": 2480203,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7996160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7906
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656414045.789, "dur": 0.211,
    "args": {
      "External id": 2480204,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8520448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7907
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656414048.687, "dur": 0.195,
    "args": {
      "External id": 2480205,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "8520704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7908
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656414049.882, "dur": 0.276,
    "args": {
      "External id": 2480206,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "9962496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7909
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656414051.360, "dur": 1.939,
    "args": {
      "External id": 2480207,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11404288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7910
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656414069.332, "dur": 26.768,
    "args": {
      "External id": 2480208,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7911
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656414157.685, "dur": 362.365,
    "args": {
      "External id": 2480209,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 7912
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514656414193.583, "dur": 321.634,
    "args": {
      "External id": 2480210,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 7913, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1336754,
    "ts": 1514656414203.519, "dur": 306.465,
    "args": {
      "External id": 2480211,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 7914
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514656414559.194, "dur": 3.142,
    "args": {
      "External id": 2480212,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 7915, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.24)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656414652.566, "dur": 17178.228,
    "args": {
      "External id": 2480213,"Record function id": 0, "Ev Idx": 7916
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656414750.187, "dur": 6.009,
    "args": {
      "External id": 2480214,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 7917
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656414763.991, "dur": 1.487,
    "args": {
      "External id": 2480215,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7918
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656414767.070, "dur": 2.190,
    "args": {
      "External id": 2480216,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7919
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656414772.535, "dur": 0.782,
    "args": {
      "External id": 2480217,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7920
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656414775.021, "dur": 1.002,
    "args": {
      "External id": 2480218,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7921
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656414777.251, "dur": 0.714,
    "args": {
      "External id": 2480219,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7922
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656414781.215, "dur": 1.067,
    "args": {
      "External id": 2480220,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7923
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656414785.261, "dur": 1.700,
    "args": {
      "External id": 2480221,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7924
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656414788.521, "dur": 0.806,
    "args": {
      "External id": 2480222,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7925
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656414790.850, "dur": 0.607,
    "args": {
      "External id": 2480223,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7926
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656414809.904, "dur": 16984.137,
    "args": {
      "External id": 2480224,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7927
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656414824.459, "dur": 16963.175,
    "args": {
      "External id": 2480225,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7928
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656414840.805, "dur": 14.463,
    "args": {
      "External id": 2480226,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7929
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514656414859.773, "dur": 16897.045,
    "args": {
      "External id": 2480227,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 7930
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656414862.145, "dur": 16894.102,
    "args": {
      "External id": 2480228,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 7931
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656414867.473, "dur": 5.104,
    "args": {
      "External id": 2480229,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7932
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656414874.148, "dur": 16878.864,
    "args": {
      "External id": 2480230,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 7933
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656431958.293, "dur": 60.633,
    "args": {
      "External id": 2480231,"Sequence number": 24740469, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 7934
    }
  },
  {
    "ph": "s", "id": 179, "pid": 1336754, "tid": 1336754, "ts": 1514656431958.293,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336754, "tid": 1336754,
    "ts": 1514656431980.621, "dur": 32.730,
    "args": {
      "External id": 2480232,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 7935
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656432008.303, "dur": 4.645,
    "args": {
      "External id": 2480233,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 7936
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336754, "tid": 1336754,
    "ts": 1514656432081.522, "dur": 75.155,
    "args": {
      "External id": 2480234,"Record function id": 0, "Ev Idx": 7937
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656432158.118, "dur": 1060.047,
    "args": {
      "External id": 2480235,"Record function id": 0, "Ev Idx": 7938
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656432197.036, "dur": 1007.925,
    "args": {
      "External id": 2480236,"Sequence number": 24740470, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 7939
    }
  },
  {
    "ph": "s", "id": 178, "pid": 1336754, "tid": 1336754, "ts": 1514656432197.036,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514656432262.436, "dur": 39.948,
    "args": {
      "External id": 2480237,"kernel_hash": "caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/ao/caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7940
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656432313.829, "dur": 105.275,
    "args": {
      "External id": 2480238,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7941
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656432428.898, "dur": 40.733,
    "args": {
      "External id": 2480239,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7942
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656432478.595, "dur": 32.236,
    "args": {
      "External id": 2480240,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7943
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656432549.293, "dur": 26.890,
    "args": {
      "External id": 2480241,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7944
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656432596.414, "dur": 15.393,
    "args": {
      "External id": 2480242,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7945
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336754, "tid": 1336754,
    "ts": 1514656432629.047, "dur": 121.566,
    "args": {
      "External id": 2480243,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 7946
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656432676.639, "dur": 11.489,
    "args": {
      "External id": 2480244,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 7947
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656432681.185, "dur": 6.163,
    "args": {
      "External id": 2480245,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7948
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656432690.539, "dur": 5.209,
    "args": {
      "External id": 2480246,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7949
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656432696.892, "dur": 1.359,
    "args": {
      "External id": 2480247,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7950
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656432700.709, "dur": 3.355,
    "args": {
      "External id": 2480248,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7951
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656432760.516, "dur": 51.798,
    "args": {
      "External id": 2480249,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7952
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336754, "tid": 1336754,
    "ts": 1514656432842.146, "dur": 26.502,
    "args": {
      "External id": 2480250,"kernel_hash": "cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/wx/cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7953
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656432877.987, "dur": 41.464,
    "args": {
      "External id": 2480251,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7954
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656432927.255, "dur": 35.103,
    "args": {
      "External id": 2480252,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7955
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514656433027.168, "dur": 30.649,
    "args": {
      "External id": 2480253,"kernel_hash": "c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2f/c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 7956
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656433066.410, "dur": 39.815,
    "args": {
      "External id": 2480254,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 7957
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656433126.731, "dur": 17.362,
    "args": {
      "External id": 2480255,"kernel_hash": "cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/gi/cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 7958
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.24)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656433279.538, "dur": 74.718,
    "args": {
      "External id": 2480256,"Record function id": 0, "Ev Idx": 7959
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656433424.942, "dur": 46.618,
    "args": {
      "External id": 2480257,"Record function id": 0, "Ev Idx": 7960
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.25)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656433480.922, "dur": 18383.295,
    "args": {
      "External id": 2480258,"Record function id": 0, "Ev Idx": 7961
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.25)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656433489.132, "dur": 926.688,
    "args": {
      "External id": 2480259,"Record function id": 0, "Ev Idx": 7962
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656433589.298, "dur": 9.053,
    "args": {
      "External id": 2480260,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7963
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514656433611.723, "dur": 32.008,
    "args": {
      "External id": 2480261,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7964
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656433616.912, "dur": 2.223,
    "args": {
      "External id": 2480262,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7965
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656433622.889, "dur": 0.205,
    "args": {
      "External id": 2480263,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7966
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656433624.371, "dur": 0.502,
    "args": {
      "External id": 2480264,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7967
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656433626.168, "dur": 0.566,
    "args": {
      "External id": 2480265,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7968
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656433629.211, "dur": 0.516,
    "args": {
      "External id": 2480266,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7969
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656433630.886, "dur": 0.338,
    "args": {
      "External id": 2480267,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7970
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656433632.563, "dur": 2.706,
    "args": {
      "External id": 2480268,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7971
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656433637.040, "dur": 0.213,
    "args": {
      "External id": 2480269,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7972
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656433638.340, "dur": 0.298,
    "args": {
      "External id": 2480270,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7973
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656433654.242, "dur": 39.200,
    "args": {
      "External id": 2480271,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7974
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1336754,
    "ts": 1514656433726.065, "dur": 101.557,
    "args": {
      "External id": 2480272,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "1", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 7975
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656433736.288, "dur": 4.489,
    "args": {
      "External id": 2480273,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7976
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1336754,
    "ts": 1514656433745.491, "dur": 10.155,
    "args": {
      "External id": 2480274,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7977
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656433750.094, "dur": 5.151,
    "args": {
      "External id": 2480275,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "12846080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 7978
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656433753.408, "dur": 0.584,
    "args": {
      "External id": 2480276,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7979
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514656433762.128, "dur": 24.736,
    "args": {
      "External id": 2480277,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7980
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656433763.859, "dur": 2.296,
    "args": {
      "External id": 2480278,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7981
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656433767.821, "dur": 0.204,
    "args": {
      "External id": 2480279,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6423296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7982
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656433769.032, "dur": 0.436,
    "args": {
      "External id": 2480280,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6947584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7983
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656433772.305, "dur": 1.202,
    "args": {
      "External id": 2480281,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7471872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7984
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656433774.316, "dur": 0.290,
    "args": {
      "External id": 2480282,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7996160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7985
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656433775.774, "dur": 0.181,
    "args": {
      "External id": 2480283,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8520448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7986
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656433778.678, "dur": 0.294,
    "args": {
      "External id": 2480284,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "8520704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7987
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656433780.008, "dur": 0.397,
    "args": {
      "External id": 2480285,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "9962496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7988
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656433781.365, "dur": 1.973,
    "args": {
      "External id": 2480286,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11404288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7989
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656433798.346, "dur": 21.557,
    "args": {
      "External id": 2480287,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7990
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656433877.772, "dur": 443.093,
    "args": {
      "External id": 2480288,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 7991
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514656433907.917, "dur": 407.235,
    "args": {
      "External id": 2480289,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 7992, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1336754,
    "ts": 1514656433917.637, "dur": 390.988,
    "args": {
      "External id": 2480290,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 7993
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514656434344.687, "dur": 2.597,
    "args": {
      "External id": 2480291,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 7994, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.25)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656434437.141, "dur": 17240.874,
    "args": {
      "External id": 2480292,"Record function id": 0, "Ev Idx": 7995
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656434549.763, "dur": 6.777,
    "args": {
      "External id": 2480293,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 7996
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656434560.560, "dur": 1.265,
    "args": {
      "External id": 2480294,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7997
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656434563.540, "dur": 2.173,
    "args": {
      "External id": 2480295,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7998
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656434567.373, "dur": 0.623,
    "args": {
      "External id": 2480296,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7999
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656434569.253, "dur": 1.073,
    "args": {
      "External id": 2480297,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 8000
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656434571.516, "dur": 0.808,
    "args": {
      "External id": 2480298,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 8001
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656434575.536, "dur": 0.835,
    "args": {
      "External id": 2480299,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 8002
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656434577.736, "dur": 1.612,
    "args": {
      "External id": 2480300,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 8003
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656434580.730, "dur": 0.809,
    "args": {
      "External id": 2480301,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 8004
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656434582.999, "dur": 0.686,
    "args": {
      "External id": 2480302,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 8005
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656434603.377, "dur": 17037.941,
    "args": {
      "External id": 2480303,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 8006
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656434619.004, "dur": 17015.366,
    "args": {
      "External id": 2480304,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 8007
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656434643.081, "dur": 14.083,
    "args": {
      "External id": 2480305,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8008
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514656434660.096, "dur": 16943.147,
    "args": {
      "External id": 2480306,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 8009
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656434662.552, "dur": 16940.253,
    "args": {
      "External id": 2480307,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 8010
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656434667.502, "dur": 5.426,
    "args": {
      "External id": 2480308,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8011
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656434674.697, "dur": 16925.121,
    "args": {
      "External id": 2480309,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 8012
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656451807.210, "dur": 33.503,
    "args": {
      "External id": 2480310,"Sequence number": 24740471, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 8013
    }
  },
  {
    "ph": "s", "id": 177, "pid": 1336754, "tid": 1336754, "ts": 1514656451807.210,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336754, "tid": 1336754,
    "ts": 1514656451829.039, "dur": 6.657,
    "args": {
      "External id": 2480311,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 8014
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656451831.584, "dur": 3.901,
    "args": {
      "External id": 2480312,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8015
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336754, "tid": 1336754,
    "ts": 1514656451899.744, "dur": 72.822,
    "args": {
      "External id": 2480313,"Record function id": 0, "Ev Idx": 8016
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656451974.182, "dur": 1056.633,
    "args": {
      "External id": 2480314,"Record function id": 0, "Ev Idx": 8017
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656452044.467, "dur": 971.802,
    "args": {
      "External id": 2480315,"Sequence number": 24740472, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 8018
    }
  },
  {
    "ph": "s", "id": 176, "pid": 1336754, "tid": 1336754, "ts": 1514656452044.467,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514656452108.253, "dur": 41.301,
    "args": {
      "External id": 2480316,"kernel_hash": "caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/ao/caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 8019
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656452161.450, "dur": 98.966,
    "args": {
      "External id": 2480317,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 8020
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656452269.597, "dur": 37.124,
    "args": {
      "External id": 2480318,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 8021
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656452315.442, "dur": 33.303,
    "args": {
      "External id": 2480319,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 8022
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656452371.360, "dur": 23.971,
    "args": {
      "External id": 2480320,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 8023
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656452410.335, "dur": 15.235,
    "args": {
      "External id": 2480321,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 8024
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336754, "tid": 1336754,
    "ts": 1514656452442.357, "dur": 138.248,
    "args": {
      "External id": 2480322,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 8025
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656452489.021, "dur": 11.118,
    "args": {
      "External id": 2480323,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8026
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656452493.677, "dur": 5.609,
    "args": {
      "External id": 2480324,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8027
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656452502.680, "dur": 4.488,
    "args": {
      "External id": 2480325,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8028
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656452508.277, "dur": 1.162,
    "args": {
      "External id": 2480326,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8029
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656452511.653, "dur": 3.286,
    "args": {
      "External id": 2480327,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8030
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656452592.030, "dur": 50.635,
    "args": {
      "External id": 2480328,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 8031
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336754, "tid": 1336754,
    "ts": 1514656452671.503, "dur": 26.642,
    "args": {
      "External id": 2480329,"kernel_hash": "cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/wx/cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 8032
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656452705.827, "dur": 43.001,
    "args": {
      "External id": 2480330,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 8033
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656452756.483, "dur": 35.877,
    "args": {
      "External id": 2480331,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 8034
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514656452813.817, "dur": 31.279,
    "args": {
      "External id": 2480332,"kernel_hash": "c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2f/c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 8035
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656452850.967, "dur": 35.118,
    "args": {
      "External id": 2480333,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 8036
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656452903.882, "dur": 16.243,
    "args": {
      "External id": 2480334,"kernel_hash": "cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/gi/cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 8037
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.25)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656453093.713, "dur": 77.847,
    "args": {
      "External id": 2480335,"Record function id": 0, "Ev Idx": 8038
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656453242.151, "dur": 47.352,
    "args": {
      "External id": 2480336,"Record function id": 0, "Ev Idx": 8039
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.26)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656453298.765, "dur": 18384.327,
    "args": {
      "External id": 2480337,"Record function id": 0, "Ev Idx": 8040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.26)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656453307.577, "dur": 902.919,
    "args": {
      "External id": 2480338,"Record function id": 0, "Ev Idx": 8041
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656453388.183, "dur": 8.657,
    "args": {
      "External id": 2480339,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8042
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514656453410.061, "dur": 32.640,
    "args": {
      "External id": 2480340,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 8043
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656453415.137, "dur": 2.336,
    "args": {
      "External id": 2480341,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8044
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656453421.468, "dur": 0.428,
    "args": {
      "External id": 2480342,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8045
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656453422.958, "dur": 0.593,
    "args": {
      "External id": 2480343,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8046
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656453424.915, "dur": 0.559,
    "args": {
      "External id": 2480344,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8047
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656453428.591, "dur": 0.573,
    "args": {
      "External id": 2480345,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8048
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656453430.230, "dur": 0.512,
    "args": {
      "External id": 2480346,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8049
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656453432.219, "dur": 2.059,
    "args": {
      "External id": 2480347,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8050
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656453435.338, "dur": 0.792,
    "args": {
      "External id": 2480348,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8051
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656453436.951, "dur": 0.338,
    "args": {
      "External id": 2480349,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8052
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656453453.684, "dur": 39.033,
    "args": {
      "External id": 2480350,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 8053
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1336754,
    "ts": 1514656453524.837, "dur": 135.749,
    "args": {
      "External id": 2480351,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "1", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 8054
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656453553.564, "dur": 5.868,
    "args": {
      "External id": 2480352,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8055
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1336754,
    "ts": 1514656453564.743, "dur": 10.684,
    "args": {
      "External id": 2480353,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 8056
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656453569.041, "dur": 5.980,
    "args": {
      "External id": 2480354,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "12846080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 8057
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656453572.617, "dur": 0.922,
    "args": {
      "External id": 2480355,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 8058
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514656453582.549, "dur": 33.605,
    "args": {
      "External id": 2480356,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 8059
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656453584.658, "dur": 2.157,
    "args": {
      "External id": 2480357,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8060
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656453587.913, "dur": 0.606,
    "args": {
      "External id": 2480358,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6423296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8061
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656453592.834, "dur": 4.808,
    "args": {
      "External id": 2480359,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6947584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8062
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656453600.923, "dur": 0.444,
    "args": {
      "External id": 2480360,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7471872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8063
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656453602.605, "dur": 0.343,
    "args": {
      "External id": 2480361,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7996160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8064
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656453604.068, "dur": 1.895,
    "args": {
      "External id": 2480362,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8520448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8065
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656453607.157, "dur": 0.566,
    "args": {
      "External id": 2480363,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "8520704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8066
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656453608.752, "dur": 0.466,
    "args": {
      "External id": 2480364,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "9962496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8067
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656453611.877, "dur": 0.494,
    "args": {
      "External id": 2480365,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11404288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8068
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656453627.566, "dur": 24.496,
    "args": {
      "External id": 2480366,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 8069
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656453715.192, "dur": 401.998,
    "args": {
      "External id": 2480367,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 8070
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514656453745.862, "dur": 366.060,
    "args": {
      "External id": 2480368,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 8071, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1336754,
    "ts": 1514656453755.497, "dur": 350.231,
    "args": {
      "External id": 2480369,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 8072
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514656454140.175, "dur": 2.814,
    "args": {
      "External id": 2480370,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 8073, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.26)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656454232.088, "dur": 17241.155,
    "args": {
      "External id": 2480371,"Record function id": 0, "Ev Idx": 8074
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656454326.849, "dur": 6.205,
    "args": {
      "External id": 2480372,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 8075
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656454336.256, "dur": 1.180,
    "args": {
      "External id": 2480373,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 8076
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656454344.734, "dur": 0.935,
    "args": {
      "External id": 2480374,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 8077
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656454348.669, "dur": 1.069,
    "args": {
      "External id": 2480375,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 8078
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656454351.308, "dur": 0.917,
    "args": {
      "External id": 2480376,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 8079
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656454353.473, "dur": 0.865,
    "args": {
      "External id": 2480377,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 8080
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656454355.933, "dur": 1.296,
    "args": {
      "External id": 2480378,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 8081
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656454360.301, "dur": 2.030,
    "args": {
      "External id": 2480379,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 8082
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656454363.909, "dur": 0.939,
    "args": {
      "External id": 2480380,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 8083
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656454366.198, "dur": 1.039,
    "args": {
      "External id": 2480381,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 8084
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656454386.006, "dur": 17049.384,
    "args": {
      "External id": 2480382,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 8085
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656454400.953, "dur": 17027.285,
    "args": {
      "External id": 2480383,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 8086
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656454417.635, "dur": 13.086,
    "args": {
      "External id": 2480384,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8087
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514656454435.489, "dur": 16960.798,
    "args": {
      "External id": 2480385,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 8088
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656454437.858, "dur": 16957.885,
    "args": {
      "External id": 2480386,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 8089
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656454443.281, "dur": 6.126,
    "args": {
      "External id": 2480387,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8090
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656454451.027, "dur": 16941.708,
    "args": {
      "External id": 2480388,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 8091
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656471622.761, "dur": 35.322,
    "args": {
      "External id": 2480389,"Sequence number": 24740473, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 8092
    }
  },
  {
    "ph": "s", "id": 175, "pid": 1336754, "tid": 1336754, "ts": 1514656471622.761,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336754, "tid": 1336754,
    "ts": 1514656471645.820, "dur": 7.450,
    "args": {
      "External id": 2480390,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 8093
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656471648.623, "dur": 4.276,
    "args": {
      "External id": 2480391,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8094
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336754, "tid": 1336754,
    "ts": 1514656471720.486, "dur": 73.725,
    "args": {
      "External id": 2480392,"Record function id": 0, "Ev Idx": 8095
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656471795.610, "dur": 1040.408,
    "args": {
      "External id": 2480393,"Record function id": 0, "Ev Idx": 8096
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656471832.937, "dur": 990.807,
    "args": {
      "External id": 2480394,"Sequence number": 24740474, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 8097
    }
  },
  {
    "ph": "s", "id": 174, "pid": 1336754, "tid": 1336754, "ts": 1514656471832.937,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514656471898.490, "dur": 40.147,
    "args": {
      "External id": 2480395,"kernel_hash": "caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/ao/caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 8098
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656471950.795, "dur": 130.380,
    "args": {
      "External id": 2480396,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 8099
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656472094.067, "dur": 41.539,
    "args": {
      "External id": 2480397,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 8100
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656472144.042, "dur": 31.986,
    "args": {
      "External id": 2480398,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 8101
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656472200.139, "dur": 26.892,
    "args": {
      "External id": 2480399,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 8102
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656472246.081, "dur": 14.774,
    "args": {
      "External id": 2480400,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 8103
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336754, "tid": 1336754,
    "ts": 1514656472277.933, "dur": 120.734,
    "args": {
      "External id": 2480401,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 8104
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656472324.857, "dur": 11.873,
    "args": {
      "External id": 2480402,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8105
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656472329.677, "dur": 6.199,
    "args": {
      "External id": 2480403,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8106
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656472339.211, "dur": 4.458,
    "args": {
      "External id": 2480404,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8107
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656472344.779, "dur": 1.613,
    "args": {
      "External id": 2480405,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8108
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656472348.631, "dur": 3.196,
    "args": {
      "External id": 2480406,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8109
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656472409.522, "dur": 45.275,
    "args": {
      "External id": 2480407,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 8110
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336754, "tid": 1336754,
    "ts": 1514656472485.294, "dur": 29.033,
    "args": {
      "External id": 2480408,"kernel_hash": "cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/wx/cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 8111
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656472522.282, "dur": 60.076,
    "args": {
      "External id": 2480409,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 8112
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656472594.487, "dur": 37.224,
    "args": {
      "External id": 2480410,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 8113
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514656472653.625, "dur": 25.965,
    "args": {
      "External id": 2480411,"kernel_hash": "c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2f/c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 8114
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656472685.612, "dur": 35.370,
    "args": {
      "External id": 2480412,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 8115
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656472740.742, "dur": 17.895,
    "args": {
      "External id": 2480413,"kernel_hash": "cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/gi/cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 8116
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.26)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656472896.991, "dur": 69.089,
    "args": {
      "External id": 2480414,"Record function id": 0, "Ev Idx": 8117
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656473078.540, "dur": 46.947,
    "args": {
      "External id": 2480415,"Record function id": 0, "Ev Idx": 8118
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.27)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656473135.863, "dur": 18380.142,
    "args": {
      "External id": 2480416,"Record function id": 0, "Ev Idx": 8119
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.27)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656473144.461, "dur": 835.271,
    "args": {
      "External id": 2480417,"Record function id": 0, "Ev Idx": 8120
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656473226.687, "dur": 9.348,
    "args": {
      "External id": 2480418,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8121
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514656473249.396, "dur": 32.357,
    "args": {
      "External id": 2480419,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 8122
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656473254.696, "dur": 2.443,
    "args": {
      "External id": 2480420,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8123
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656473261.021, "dur": 0.431,
    "args": {
      "External id": 2480421,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8124
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656473262.919, "dur": 0.683,
    "args": {
      "External id": 2480422,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8125
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656473264.807, "dur": 0.644,
    "args": {
      "External id": 2480423,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8126
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656473267.998, "dur": 0.749,
    "args": {
      "External id": 2480424,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8127
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656473269.637, "dur": 0.538,
    "args": {
      "External id": 2480425,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8128
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656473270.914, "dur": 1.902,
    "args": {
      "External id": 2480426,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8129
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656473274.020, "dur": 0.540,
    "args": {
      "External id": 2480427,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8130
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656473275.717, "dur": 0.554,
    "args": {
      "External id": 2480428,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8131
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656473292.008, "dur": 39.247,
    "args": {
      "External id": 2480429,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 8132
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336754, "tid": 1336754,
    "ts": 1514656473363.916, "dur": 104.869,
    "args": {
      "External id": 2480430,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "1", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 8133
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656473374.227, "dur": 3.984,
    "args": {
      "External id": 2480431,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8134
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336754, "tid": 1336754,
    "ts": 1514656473382.833, "dur": 9.786,
    "args": {
      "External id": 2480432,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 8135
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656473387.008, "dur": 5.196,
    "args": {
      "External id": 2480433,"Record function id": 0, "Concrete Inputs": ["", "0", "6423040", "12846080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 8136
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656473390.198, "dur": 0.809,
    "args": {
      "External id": 2480434,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 8137
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336754, "tid": 1336754,
    "ts": 1514656473399.667, "dur": 26.996,
    "args": {
      "External id": 2480435,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 8138
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656473401.912, "dur": 1.979,
    "args": {
      "External id": 2480436,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "6423040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8139
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656473405.269, "dur": 0.783,
    "args": {
      "External id": 2480437,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6423296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8140
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656473407.275, "dur": 0.531,
    "args": {
      "External id": 2480438,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "6947584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8141
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656473410.574, "dur": 0.733,
    "args": {
      "External id": 2480439,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7471872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8142
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656473412.316, "dur": 0.781,
    "args": {
      "External id": 2480440,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "7996160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8143
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656473414.088, "dur": 0.599,
    "args": {
      "External id": 2480441,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8520448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8144
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656473417.433, "dur": 0.743,
    "args": {
      "External id": 2480442,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "8520704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8145
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656473419.339, "dur": 0.555,
    "args": {
      "External id": 2480443,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "9962496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8146
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656473420.820, "dur": 2.222,
    "args": {
      "External id": 2480444,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11404288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8147
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656473437.974, "dur": 23.162,
    "args": {
      "External id": 2480445,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 8148
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656473519.080, "dur": 369.042,
    "args": {
      "External id": 2480446,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 8149
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514656473565.426, "dur": 317.495,
    "args": {
      "External id": 2480447,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 1, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 8150, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336754, "tid": 1336754,
    "ts": 1514656473576.245, "dur": 300.024,
    "args": {
      "External id": 2480448,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 8151
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514656473909.464, "dur": 2.447,
    "args": {
      "External id": 2480449,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 8152, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.27)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656474039.291, "dur": 17282.125,
    "args": {
      "External id": 2480450,"Record function id": 0, "Ev Idx": 8153
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656474154.277, "dur": 6.614,
    "args": {
      "External id": 2480451,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 8154
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656474164.353, "dur": 1.332,
    "args": {
      "External id": 2480452,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 8155
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656474167.403, "dur": 1.131,
    "args": {
      "External id": 2480453,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 8156
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656474171.874, "dur": 1.170,
    "args": {
      "External id": 2480454,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 8157
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656474174.560, "dur": 0.981,
    "args": {
      "External id": 2480455,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 8158
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656474176.820, "dur": 0.954,
    "args": {
      "External id": 2480456,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 8159
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656474179.441, "dur": 1.066,
    "args": {
      "External id": 2480457,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 8160
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656474183.589, "dur": 1.962,
    "args": {
      "External id": 2480458,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 8161
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656474186.777, "dur": 1.215,
    "args": {
      "External id": 2480459,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 8162
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656474189.631, "dur": 0.984,
    "args": {
      "External id": 2480460,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 8163
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656474209.761, "dur": 17072.333,
    "args": {
      "External id": 2480461,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 8164
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656474224.745, "dur": 17050.363,
    "args": {
      "External id": 2480462,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 8165
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656474241.279, "dur": 13.576,
    "args": {
      "External id": 2480463,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8166
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514656474259.888, "dur": 16983.510,
    "args": {
      "External id": 2480464,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 8167
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656474262.501, "dur": 16980.323,
    "args": {
      "External id": 2480465,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 8168
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656474267.538, "dur": 5.788,
    "args": {
      "External id": 2480466,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8169
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656474275.044, "dur": 16964.801,
    "args": {
      "External id": 2480467,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 8170
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656491455.730, "dur": 36.208,
    "args": {
      "External id": 2480468,"Sequence number": 24740475, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 8171
    }
  },
  {
    "ph": "s", "id": 173, "pid": 1336754, "tid": 1336754, "ts": 1514656491455.730,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336754, "tid": 1336754,
    "ts": 1514656491479.590, "dur": 7.216,
    "args": {
      "External id": 2480469,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 8172
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656491482.474, "dur": 4.146,
    "args": {
      "External id": 2480470,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8173
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336754, "tid": 1336754,
    "ts": 1514656491566.407, "dur": 74.711,
    "args": {
      "External id": 2480471,"Record function id": 0, "Ev Idx": 8174
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656491643.052, "dur": 1054.525,
    "args": {
      "External id": 2480472,"Record function id": 0, "Ev Idx": 8175
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656491681.389, "dur": 1002.859,
    "args": {
      "External id": 2480473,"Sequence number": 24740476, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 8176
    }
  },
  {
    "ph": "s", "id": 172, "pid": 1336754, "tid": 1336754, "ts": 1514656491681.389,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514656491746.978, "dur": 40.977,
    "args": {
      "External id": 2480474,"kernel_hash": "caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/ao/caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 8177
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656491801.110, "dur": 104.670,
    "args": {
      "External id": 2480475,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 8178
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656491914.053, "dur": 39.349,
    "args": {
      "External id": 2480476,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 8179
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656491961.522, "dur": 63.146,
    "args": {
      "External id": 2480477,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 8180
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656492053.392, "dur": 26.116,
    "args": {
      "External id": 2480478,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 8181
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656492097.853, "dur": 14.570,
    "args": {
      "External id": 2480479,"kernel_hash": "cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vf/cvfmx3dtueggejegez27w222l5wve5udft5w5bpqit5metyl3lao.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 8182
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336754, "tid": 1336754,
    "ts": 1514656492129.651, "dur": 123.085,
    "args": {
      "External id": 2480480,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 8183
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656492178.454, "dur": 11.288,
    "args": {
      "External id": 2480481,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8184
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656492183.174, "dur": 5.740,
    "args": {
      "External id": 2480482,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8185
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656492192.178, "dur": 4.554,
    "args": {
      "External id": 2480483,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8186
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656492197.818, "dur": 1.565,
    "args": {
      "External id": 2480484,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8187
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656492201.804, "dur": 2.846,
    "args": {
      "External id": 2480485,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8188
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656492263.655, "dur": 50.349,
    "args": {
      "External id": 2480486,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 8189
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336754, "tid": 1336754,
    "ts": 1514656492344.691, "dur": 28.219,
    "args": {
      "External id": 2480487,"kernel_hash": "cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/wx/cwxmlnjvnrmqdfwpkx5pm5ox5uexmxff6wgrjt6ou3pvccpt3bci.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 8190
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656492380.602, "dur": 41.251,
    "args": {
      "External id": 2480488,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 8191
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656492429.914, "dur": 35.883,
    "args": {
      "External id": 2480489,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 8192
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514656492486.665, "dur": 27.980,
    "args": {
      "External id": 2480490,"kernel_hash": "c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2f/c2fsjwk3nwbfke7oj7nq5k7qop6mvatbrirsxktytkvp2mdmme43.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 8193
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656492520.630, "dur": 52.515,
    "args": {
      "External id": 2480491,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 8194
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336754, "tid": 1336754,
    "ts": 1514656492598.578, "dur": 19.253,
    "args": {
      "External id": 2480492,"kernel_hash": "cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/gi/cgik4ktlpri5iod5s5ynrzau3qlffebgfpe6sdr2lepxbydsdryb.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 8195
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.27)", "pid": 1336754, "tid": 1336754,
    "ts": 1514656492758.680, "dur": 32.101,
    "args": {
      "External id": 2480493,"Record function id": 0, "Ev Idx": 8196
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656492919.610, "dur": 329.423,
    "args": {
      "External id": 2480494,"Sequence number": 24740477, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "9.9999999999999995e-07", "False", "False", "True"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [1], [], [], [], [], [], []], "Input Dims": [[16, 4096, 2048], [2048], [], [], [], [], [], []], "Ev Idx": 8197
    }
  },
  {
    "ph": "s", "id": 171, "pid": 1336754, "tid": 1336754, "ts": 1514656492919.610,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656492951.207, "dur": 7.559,
    "args": {
      "External id": 2480495,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8198
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656492953.385, "dur": 5.067,
    "args": {
      "External id": 2480496,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8199
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656492968.807, "dur": 11.333,
    "args": {
      "External id": 2480497,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 8200
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656492971.688, "dur": 7.883,
    "args": {
      "External id": 2480498,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8201
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656493025.049, "dur": 6.057,
    "args": {
      "External id": 2480499,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8202
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656493225.960, "dur": 8.498,
    "args": {
      "External id": 2480500,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8203
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656493230.419, "dur": 3.750,
    "args": {
      "External id": 2480501,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8204
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656493279.227, "dur": 134.777,
    "args": {
      "External id": 2480502,"Sequence number": 24740478, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8205
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656493282.865, "dur": 15.446,
    "args": {
      "External id": 2480503,"Sequence number": 24740478, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8206
    }
  },
  {
    "ph": "s", "id": 170, "pid": 1336754, "tid": 1336754, "ts": 1514656493282.865,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656493288.525, "dur": 7.776,
    "args": {
      "External id": 2480504,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8207
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656493293.513, "dur": 2.402,
    "args": {
      "External id": 2480505,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8208
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656493301.471, "dur": 112.147,
    "args": {
      "External id": 2480506,"Sequence number": 24740479, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8209
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656493303.900, "dur": 4.370,
    "args": {
      "External id": 2480507,"Sequence number": 24740479, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8210
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656493304.709, "dur": 3.424,
    "args": {
      "External id": 2480508,"Sequence number": 24740479, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8211
    }
  },
  {
    "ph": "s", "id": 169, "pid": 1336754, "tid": 1336754, "ts": 1514656493304.709,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656493311.682, "dur": 92.479,
    "args": {
      "External id": 2480509,"Sequence number": 24740480, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8212
    }
  },
  {
    "ph": "s", "id": 168, "pid": 1336754, "tid": 1336754, "ts": 1514656493311.682,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656493408.171, "dur": 4.551,
    "args": {
      "External id": 2480510,"Sequence number": 24740481, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8213
    }
  },
  {
    "ph": "s", "id": 167, "pid": 1336754, "tid": 1336754, "ts": 1514656493408.171,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656493423.747, "dur": 68.363,
    "args": {
      "External id": 2480511,"Sequence number": 24740482, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8214
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656493424.618, "dur": 7.301,
    "args": {
      "External id": 2480512,"Sequence number": 24740482, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8215
    }
  },
  {
    "ph": "s", "id": 166, "pid": 1336754, "tid": 1336754, "ts": 1514656493424.618,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656493426.468, "dur": 4.388,
    "args": {
      "External id": 2480513,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8216
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656493429.718, "dur": 0.941,
    "args": {
      "External id": 2480514,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8217
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656493432.725, "dur": 59.110,
    "args": {
      "External id": 2480515,"Sequence number": 24740483, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8218
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656493433.677, "dur": 4.689,
    "args": {
      "External id": 2480516,"Sequence number": 24740483, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8219
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656493434.444, "dur": 3.712,
    "args": {
      "External id": 2480517,"Sequence number": 24740483, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8220
    }
  },
  {
    "ph": "s", "id": 165, "pid": 1336754, "tid": 1336754, "ts": 1514656493434.444,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656493438.989, "dur": 45.959,
    "args": {
      "External id": 2480518,"Sequence number": 24740484, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8221
    }
  },
  {
    "ph": "s", "id": 164, "pid": 1336754, "tid": 1336754, "ts": 1514656493438.989,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656493486.916, "dur": 4.459,
    "args": {
      "External id": 2480519,"Sequence number": 24740485, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8222
    }
  },
  {
    "ph": "s", "id": 163, "pid": 1336754, "tid": 1336754, "ts": 1514656493486.916,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656493499.559, "dur": 85.955,
    "args": {
      "External id": 2480520,"Sequence number": 24740486, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8223
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656493500.282, "dur": 4.905,
    "args": {
      "External id": 2480521,"Sequence number": 24740486, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8224
    }
  },
  {
    "ph": "s", "id": 162, "pid": 1336754, "tid": 1336754, "ts": 1514656493500.282,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656493501.627, "dur": 2.439,
    "args": {
      "External id": 2480522,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8225
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656493503.027, "dur": 0.713,
    "args": {
      "External id": 2480523,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8226
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656493507.352, "dur": 77.932,
    "args": {
      "External id": 2480524,"Sequence number": 24740487, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8227
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656493508.279, "dur": 4.218,
    "args": {
      "External id": 2480525,"Sequence number": 24740487, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8228
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656493508.939, "dur": 3.420,
    "args": {
      "External id": 2480526,"Sequence number": 24740487, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8229
    }
  },
  {
    "ph": "s", "id": 161, "pid": 1336754, "tid": 1336754, "ts": 1514656493508.939,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656493513.015, "dur": 63.666,
    "args": {
      "External id": 2480527,"Sequence number": 24740488, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8230
    }
  },
  {
    "ph": "s", "id": 160, "pid": 1336754, "tid": 1336754, "ts": 1514656493513.015,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656493579.563, "dur": 5.275,
    "args": {
      "External id": 2480528,"Sequence number": 24740489, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8231
    }
  },
  {
    "ph": "s", "id": 159, "pid": 1336754, "tid": 1336754, "ts": 1514656493579.563,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656493608.016, "dur": 4.091,
    "args": {
      "External id": 2480529,"Sequence number": 24740490, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8232
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656493608.955, "dur": 3.009,
    "args": {
      "External id": 2480530,"Sequence number": 24740490, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8233
    }
  },
  {
    "ph": "s", "id": 158, "pid": 1336754, "tid": 1336754, "ts": 1514656493608.955,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656493618.661, "dur": 3.070,
    "args": {
      "External id": 2480531,"Sequence number": 24740491, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8234
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656493619.461, "dur": 2.143,
    "args": {
      "External id": 2480532,"Sequence number": 24740491, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8235
    }
  },
  {
    "ph": "s", "id": 157, "pid": 1336754, "tid": 1336754, "ts": 1514656493619.461,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656493627.301, "dur": 4.766,
    "args": {
      "External id": 2480533,"Sequence number": 24740492, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8236
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656493628.161, "dur": 3.760,
    "args": {
      "External id": 2480534,"Sequence number": 24740492, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8237
    }
  },
  {
    "ph": "s", "id": 156, "pid": 1336754, "tid": 1336754, "ts": 1514656493628.161,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656493667.499, "dur": 174.632,
    "args": {
      "External id": 2480535,"Sequence number": 24740493, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "False", "False", "0", "", "8192"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [], [], [], [], []], "Ev Idx": 8238
    }
  },
  {
    "ph": "s", "id": 155, "pid": 1336754, "tid": 1336754, "ts": 1514656493667.499,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656493690.262, "dur": 9.109,
    "args": {
      "External id": 2480536,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8239
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656493693.058, "dur": 5.764,
    "args": {
      "External id": 2480537,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8240
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656493854.666, "dur": 105.669,
    "args": {
      "External id": 2480538,"Sequence number": 24740494, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "False", "False", "0", "", "8192"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [], [], [], [], []], "Ev Idx": 8241
    }
  },
  {
    "ph": "s", "id": 154, "pid": 1336754, "tid": 1336754, "ts": 1514656493854.666,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656493869.819, "dur": 6.289,
    "args": {
      "External id": 2480539,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8242
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656493871.702, "dur": 4.079,
    "args": {
      "External id": 2480540,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8243
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FlashAttnFunc", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494024.279, "dur": 191.347,
    "args": {
      "External id": 2480541,"Sequence number": 24740495, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "", "True", "", "0.", "", "False", "False", "True"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "", "Scalar", "", "Scalar", "", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], []], "Ev Idx": 8244
    }
  },
  {
    "ph": "s", "id": 153, "pid": 1336754, "tid": 1336754, "ts": 1514656494024.279,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494057.529, "dur": 127.064,
    "args": {
      "External id": 2480542,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 8245
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494106.322, "dur": 7.809,
    "args": {
      "External id": 2480543,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8246
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494108.505, "dur": 5.090,
    "args": {
      "External id": 2480544,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8247
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494116.848, "dur": 4.272,
    "args": {
      "External id": 2480545,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8248
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494122.355, "dur": 1.564,
    "args": {
      "External id": 2480546,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8249
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494127.961, "dur": 2.924,
    "args": {
      "External id": 2480547,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8250
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494199.162, "dur": 5.527,
    "args": {
      "External id": 2480548,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 8251
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494221.185, "dur": 5.897,
    "args": {
      "External id": 2480549,"Sequence number": 24740496, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 8252
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494222.406, "dur": 4.449,
    "args": {
      "External id": 2480550,"Sequence number": 24740496, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 8253
    }
  },
  {
    "ph": "s", "id": 152, "pid": 1336754, "tid": 1336754, "ts": 1514656494222.406,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494243.923, "dur": 128.382,
    "args": {
      "External id": 2480551,"Sequence number": 24740497, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8254
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494246.957, "dur": 8.287,
    "args": {
      "External id": 2480552,"Sequence number": 24740497, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8255
    }
  },
  {
    "ph": "s", "id": 151, "pid": 1336754, "tid": 1336754, "ts": 1514656494246.957,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494249.439, "dur": 4.546,
    "args": {
      "External id": 2480553,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8256
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494251.803, "dur": 1.862,
    "args": {
      "External id": 2480554,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8257
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494256.704, "dur": 115.331,
    "args": {
      "External id": 2480555,"Sequence number": 24740498, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8258
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494258.438, "dur": 7.960,
    "args": {
      "External id": 2480556,"Sequence number": 24740498, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8259
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494260.834, "dur": 5.391,
    "args": {
      "External id": 2480557,"Sequence number": 24740498, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8260
    }
  },
  {
    "ph": "s", "id": 150, "pid": 1336754, "tid": 1336754, "ts": 1514656494260.834,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494267.365, "dur": 96.706,
    "args": {
      "External id": 2480558,"Sequence number": 24740499, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8261
    }
  },
  {
    "ph": "s", "id": 149, "pid": 1336754, "tid": 1336754, "ts": 1514656494267.365,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494366.475, "dur": 4.846,
    "args": {
      "External id": 2480559,"Sequence number": 24740500, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8262
    }
  },
  {
    "ph": "s", "id": 148, "pid": 1336754, "tid": 1336754, "ts": 1514656494366.475,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494406.200, "dur": 237.602,
    "args": {
      "External id": 2480560,"Sequence number": 24740501, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "9.9999999999999995e-07", "True", "False", "True"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [1], [], [8388608, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 2048], [2048], [], [16, 4096, 2048], [], [], [], []], "Ev Idx": 8263
    }
  },
  {
    "ph": "s", "id": 147, "pid": 1336754, "tid": 1336754, "ts": 1514656494406.200,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494425.712, "dur": 2.668,
    "args": {
      "External id": 2480561,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8264
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494426.545, "dur": 1.630,
    "args": {
      "External id": 2480562,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8265
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape_as", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494433.150, "dur": 3.244,
    "args": {
      "External id": 2480563,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [2048, 1]], "Input Dims": [[16, 4096, 2048], [65536, 2048]], "Ev Idx": 8266
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494434.267, "dur": 2.019,
    "args": {
      "External id": 2480564,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8267
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494435.006, "dur": 1.167,
    "args": {
      "External id": 2480565,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8268
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494443.769, "dur": 8.281,
    "args": {
      "External id": 2480566,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 8269
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494446.848, "dur": 4.725,
    "args": {
      "External id": 2480567,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8270
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494457.873, "dur": 3.215,
    "args": {
      "External id": 2480568,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8271
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494464.749, "dur": 2.768,
    "args": {
      "External id": 2480569,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8272
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494617.630, "dur": 4.151,
    "args": {
      "External id": 2480570,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8273
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494618.573, "dur": 2.874,
    "args": {
      "External id": 2480571,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8274
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494624.461, "dur": 2.439,
    "args": {
      "External id": 2480572,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8275
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494625.279, "dur": 1.507,
    "args": {
      "External id": 2480573,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8276
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494664.624, "dur": 96.049,
    "args": {
      "External id": 2480574,"Sequence number": 24740502, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [5632, 2048], []], "Ev Idx": 8277
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494665.796, "dur": 7.919,
    "args": {
      "External id": 2480575,"Sequence number": 24740502, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 8278
    }
  },
  {
    "ph": "s", "id": 146, "pid": 1336754, "tid": 1336754, "ts": 1514656494665.796,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494668.368, "dur": 4.090,
    "args": {
      "External id": 2480576,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 8279
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494670.601, "dur": 1.512,
    "args": {
      "External id": 2480577,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 8280
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494674.613, "dur": 85.744,
    "args": {
      "External id": 2480578,"Sequence number": 24740503, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 5632]], "Ev Idx": 8281
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494677.799, "dur": 3.091,
    "args": {
      "External id": 2480579,"Sequence number": 24740503, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8282
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494678.598, "dur": 2.169,
    "args": {
      "External id": 2480580,"Sequence number": 24740503, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8283
    }
  },
  {
    "ph": "s", "id": 145, "pid": 1336754, "tid": 1336754, "ts": 1514656494678.598,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494681.551, "dur": 73.039,
    "args": {
      "External id": 2480581,"Sequence number": 24740504, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 8284
    }
  },
  {
    "ph": "s", "id": 144, "pid": 1336754, "tid": 1336754, "ts": 1514656494681.551,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494756.979, "dur": 2.810,
    "args": {
      "External id": 2480582,"Sequence number": 24740505, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 8285
    }
  },
  {
    "ph": "s", "id": 143, "pid": 1336754, "tid": 1336754, "ts": 1514656494756.979,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494769.775, "dur": 71.182,
    "args": {
      "External id": 2480583,"Sequence number": 24740506, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [5632, 2048], []], "Ev Idx": 8286
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494772.424, "dur": 6.052,
    "args": {
      "External id": 2480584,"Sequence number": 24740506, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 8287
    }
  },
  {
    "ph": "s", "id": 142, "pid": 1336754, "tid": 1336754, "ts": 1514656494772.424,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494774.270, "dur": 3.088,
    "args": {
      "External id": 2480585,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 8288
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494776.018, "dur": 1.186,
    "args": {
      "External id": 2480586,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 8289
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494779.078, "dur": 61.598,
    "args": {
      "External id": 2480587,"Sequence number": 24740507, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 5632]], "Ev Idx": 8290
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494780.186, "dur": 7.436,
    "args": {
      "External id": 2480588,"Sequence number": 24740507, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8291
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494782.701, "dur": 4.768,
    "args": {
      "External id": 2480589,"Sequence number": 24740507, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8292
    }
  },
  {
    "ph": "s", "id": 141, "pid": 1336754, "tid": 1336754, "ts": 1514656494782.701,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494788.224, "dur": 46.301,
    "args": {
      "External id": 2480590,"Sequence number": 24740508, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 8293
    }
  },
  {
    "ph": "s", "id": 140, "pid": 1336754, "tid": 1336754, "ts": 1514656494788.224,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494836.119, "dur": 4.230,
    "args": {
      "External id": 2480591,"Sequence number": 24740509, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 8294
    }
  },
  {
    "ph": "s", "id": 139, "pid": 1336754, "tid": 1336754, "ts": 1514656494836.119,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SwiGLULinearFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494863.062, "dur": 194.476,
    "args": {
      "External id": 2480592,"Sequence number": 24740510, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[23068672, 5632, 1], [23068672, 5632, 1], [5632, 1], []], "Input Dims": [[16, 4096, 5632], [16, 4096, 5632], [2048, 5632], []], "Ev Idx": 8295
    }
  },
  {
    "ph": "s", "id": 138, "pid": 1336754, "tid": 1336754, "ts": 1514656494863.062,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494907.988, "dur": 4.772,
    "args": {
      "External id": 2480593,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8296
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494946.403, "dur": 95.690,
    "args": {
      "External id": 2480594,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[23068672, 5632, 1], [5632, 1], []], "Input Dims": [[16, 4096, 5632], [2048, 5632], []], "Ev Idx": 8297
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494947.193, "dur": 5.072,
    "args": {
      "External id": 2480595,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 8298
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494948.229, "dur": 3.293,
    "args": {
      "External id": 2480596,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 8299
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494949.998, "dur": 1.205,
    "args": {
      "External id": 2480597,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 8300
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494953.073, "dur": 88.640,
    "args": {
      "External id": 2480598,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[23068672, 5632, 1], [1, 5632]], "Input Dims": [[16, 4096, 5632], [5632, 2048]], "Ev Idx": 8301
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494955.723, "dur": 2.010,
    "args": {
      "External id": 2480599,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 8302
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494956.509, "dur": 1.074,
    "args": {
      "External id": 2480600,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 8303
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656494958.339, "dur": 77.523,
    "args": {
      "External id": 2480601,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632]], "Input Dims": [[65536, 5632], [5632, 2048]], "Ev Idx": 8304
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656495039.544, "dur": 1.450,
    "args": {
      "External id": 2480602,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8305
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336754, "tid": 1336754,
    "ts": 1514656495068.426, "dur": 27.923,
    "args": {
      "External id": 2480603,"Sequence number": 24740511, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 8306
    }
  },
  {
    "ph": "s", "id": 137, "pid": 1336754, "tid": 1336754, "ts": 1514656495068.426,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656495133.169, "dur": 179.445,
    "args": {
      "External id": 2480604,"Sequence number": 24740512, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "9.9999999999999995e-07", "False", "False", "True"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [1], [], [], [], [], [], []], "Input Dims": [[16, 4096, 2048], [2048], [], [], [], [], [], []], "Ev Idx": 8307
    }
  },
  {
    "ph": "s", "id": 136, "pid": 1336754, "tid": 1336754, "ts": 1514656495133.169,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656495153.662, "dur": 5.084,
    "args": {
      "External id": 2480605,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8308
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656495156.511, "dur": 1.964,
    "args": {
      "External id": 2480606,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8309
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656495166.821, "dur": 6.953,
    "args": {
      "External id": 2480607,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 8310
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656495169.335, "dur": 4.020,
    "args": {
      "External id": 2480608,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8311
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656495179.624, "dur": 3.951,
    "args": {
      "External id": 2480609,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8312
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656495298.933, "dur": 3.008,
    "args": {
      "External id": 2480610,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8313
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656495299.856, "dur": 1.864,
    "args": {
      "External id": 2480611,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8314
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656495330.856, "dur": 89.551,
    "args": {
      "External id": 2480612,"Sequence number": 24740513, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8315
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656495331.881, "dur": 8.734,
    "args": {
      "External id": 2480613,"Sequence number": 24740513, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8316
    }
  },
  {
    "ph": "s", "id": 135, "pid": 1336754, "tid": 1336754, "ts": 1514656495331.881,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656495335.251, "dur": 3.855,
    "args": {
      "External id": 2480614,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8317
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656495337.121, "dur": 1.735,
    "args": {
      "External id": 2480615,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8318
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656495341.524, "dur": 78.581,
    "args": {
      "External id": 2480616,"Sequence number": 24740514, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8319
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656495342.957, "dur": 5.761,
    "args": {
      "External id": 2480617,"Sequence number": 24740514, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8320
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656495345.738, "dur": 2.847,
    "args": {
      "External id": 2480618,"Sequence number": 24740514, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8321
    }
  },
  {
    "ph": "s", "id": 134, "pid": 1336754, "tid": 1336754, "ts": 1514656495345.738,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656495349.483, "dur": 65.548,
    "args": {
      "External id": 2480619,"Sequence number": 24740515, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8322
    }
  },
  {
    "ph": "s", "id": 133, "pid": 1336754, "tid": 1336754, "ts": 1514656495349.483,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656495416.945, "dur": 2.624,
    "args": {
      "External id": 2480620,"Sequence number": 24740516, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8323
    }
  },
  {
    "ph": "s", "id": 132, "pid": 1336754, "tid": 1336754, "ts": 1514656495416.945,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656495428.356, "dur": 62.830,
    "args": {
      "External id": 2480621,"Sequence number": 24740517, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8324
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656495428.852, "dur": 8.645,
    "args": {
      "External id": 2480622,"Sequence number": 24740517, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8325
    }
  },
  {
    "ph": "s", "id": 131, "pid": 1336754, "tid": 1336754, "ts": 1514656495428.852,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656495434.112, "dur": 2.330,
    "args": {
      "External id": 2480623,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8326
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656495435.385, "dur": 0.864,
    "args": {
      "External id": 2480624,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8327
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656495438.057, "dur": 52.902,
    "args": {
      "External id": 2480625,"Sequence number": 24740518, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8328
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656495439.212, "dur": 4.343,
    "args": {
      "External id": 2480626,"Sequence number": 24740518, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8329
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656495439.985, "dur": 3.452,
    "args": {
      "External id": 2480627,"Sequence number": 24740518, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8330
    }
  },
  {
    "ph": "s", "id": 130, "pid": 1336754, "tid": 1336754, "ts": 1514656495439.985,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656495444.210, "dur": 42.969,
    "args": {
      "External id": 2480628,"Sequence number": 24740519, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8331
    }
  },
  {
    "ph": "s", "id": 129, "pid": 1336754, "tid": 1336754, "ts": 1514656495444.210,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656495488.645, "dur": 2.008,
    "args": {
      "External id": 2480629,"Sequence number": 24740520, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8332
    }
  },
  {
    "ph": "s", "id": 128, "pid": 1336754, "tid": 1336754, "ts": 1514656495488.645,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656495497.829, "dur": 81.323,
    "args": {
      "External id": 2480630,"Sequence number": 24740521, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8333
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656495498.564, "dur": 8.397,
    "args": {
      "External id": 2480631,"Sequence number": 24740521, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8334
    }
  },
  {
    "ph": "s", "id": 127, "pid": 1336754, "tid": 1336754, "ts": 1514656495498.564,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656495500.170, "dur": 5.674,
    "args": {
      "External id": 2480632,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8335
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656495503.306, "dur": 2.327,
    "args": {
      "External id": 2480633,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8336
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656495507.530, "dur": 71.322,
    "args": {
      "External id": 2480634,"Sequence number": 24740522, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8337
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656495508.316, "dur": 4.217,
    "args": {
      "External id": 2480635,"Sequence number": 24740522, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8338
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656495509.250, "dur": 3.166,
    "args": {
      "External id": 2480636,"Sequence number": 24740522, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8339
    }
  },
  {
    "ph": "s", "id": 126, "pid": 1336754, "tid": 1336754, "ts": 1514656495509.250,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656495514.802, "dur": 56.517,
    "args": {
      "External id": 2480637,"Sequence number": 24740523, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8340
    }
  },
  {
    "ph": "s", "id": 125, "pid": 1336754, "tid": 1336754, "ts": 1514656495514.802,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656495574.289, "dur": 4.106,
    "args": {
      "External id": 2480638,"Sequence number": 24740524, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8341
    }
  },
  {
    "ph": "s", "id": 124, "pid": 1336754, "tid": 1336754, "ts": 1514656495574.289,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656495595.502, "dur": 4.349,
    "args": {
      "External id": 2480639,"Sequence number": 24740525, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8342
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656495596.496, "dur": 3.061,
    "args": {
      "External id": 2480640,"Sequence number": 24740525, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8343
    }
  },
  {
    "ph": "s", "id": 123, "pid": 1336754, "tid": 1336754, "ts": 1514656495596.496,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656495630.933, "dur": 6.270,
    "args": {
      "External id": 2480641,"Sequence number": 24740526, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8344
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656495633.934, "dur": 3.092,
    "args": {
      "External id": 2480642,"Sequence number": 24740526, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8345
    }
  },
  {
    "ph": "s", "id": 122, "pid": 1336754, "tid": 1336754, "ts": 1514656495633.934,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656495641.643, "dur": 3.733,
    "args": {
      "External id": 2480643,"Sequence number": 24740527, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8346
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656495642.242, "dur": 2.968,
    "args": {
      "External id": 2480644,"Sequence number": 24740527, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8347
    }
  },
  {
    "ph": "s", "id": 121, "pid": 1336754, "tid": 1336754, "ts": 1514656495642.242,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656495674.001, "dur": 153.844,
    "args": {
      "External id": 2480645,"Sequence number": 24740528, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "False", "False", "0", "", "8192"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [], [], [], [], []], "Ev Idx": 8348
    }
  },
  {
    "ph": "s", "id": 120, "pid": 1336754, "tid": 1336754, "ts": 1514656495674.001,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656495693.872, "dur": 8.248,
    "args": {
      "External id": 2480646,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8349
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656495696.188, "dur": 5.537,
    "args": {
      "External id": 2480647,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8350
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656495841.027, "dur": 99.505,
    "args": {
      "External id": 2480648,"Sequence number": 24740529, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "False", "False", "0", "", "8192"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [], [], [], [], []], "Ev Idx": 8351
    }
  },
  {
    "ph": "s", "id": 119, "pid": 1336754, "tid": 1336754, "ts": 1514656495841.027,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656495855.784, "dur": 6.251,
    "args": {
      "External id": 2480649,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8352
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656495857.697, "dur": 3.940,
    "args": {
      "External id": 2480650,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8353
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FlashAttnFunc", "pid": 1336754, "tid": 1336754,
    "ts": 1514656495966.707, "dur": 226.290,
    "args": {
      "External id": 2480651,"Sequence number": 24740530, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "", "True", "", "0.", "", "False", "False", "True"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "", "Scalar", "", "Scalar", "", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], []], "Ev Idx": 8354
    }
  },
  {
    "ph": "s", "id": 118, "pid": 1336754, "tid": 1336754, "ts": 1514656495966.707,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496042.997, "dur": 123.896,
    "args": {
      "External id": 2480652,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 8355
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496090.382, "dur": 8.398,
    "args": {
      "External id": 2480653,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8356
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496092.588, "dur": 5.558,
    "args": {
      "External id": 2480654,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8357
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496101.227, "dur": 4.560,
    "args": {
      "External id": 2480655,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8358
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496107.025, "dur": 3.170,
    "args": {
      "External id": 2480656,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8359
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496112.357, "dur": 3.248,
    "args": {
      "External id": 2480657,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8360
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496177.771, "dur": 5.151,
    "args": {
      "External id": 2480658,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 8361
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496198.077, "dur": 6.978,
    "args": {
      "External id": 2480659,"Sequence number": 24740531, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 8362
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496199.015, "dur": 5.907,
    "args": {
      "External id": 2480660,"Sequence number": 24740531, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 8363
    }
  },
  {
    "ph": "s", "id": 117, "pid": 1336754, "tid": 1336754, "ts": 1514656496199.015,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496220.649, "dur": 112.271,
    "args": {
      "External id": 2480661,"Sequence number": 24740532, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8364
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496221.783, "dur": 13.567,
    "args": {
      "External id": 2480662,"Sequence number": 24740532, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8365
    }
  },
  {
    "ph": "s", "id": 116, "pid": 1336754, "tid": 1336754, "ts": 1514656496221.783,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496224.152, "dur": 10.093,
    "args": {
      "External id": 2480663,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8366
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496232.251, "dur": 1.695,
    "args": {
      "External id": 2480664,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8367
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496236.423, "dur": 96.100,
    "args": {
      "External id": 2480665,"Sequence number": 24740533, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8368
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496239.546, "dur": 4.777,
    "args": {
      "External id": 2480666,"Sequence number": 24740533, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8369
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496240.176, "dur": 3.993,
    "args": {
      "External id": 2480667,"Sequence number": 24740533, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8370
    }
  },
  {
    "ph": "s", "id": 115, "pid": 1336754, "tid": 1336754, "ts": 1514656496240.176,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496245.194, "dur": 79.544,
    "args": {
      "External id": 2480668,"Sequence number": 24740534, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8371
    }
  },
  {
    "ph": "s", "id": 114, "pid": 1336754, "tid": 1336754, "ts": 1514656496245.194,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496327.052, "dur": 4.760,
    "args": {
      "External id": 2480669,"Sequence number": 24740535, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8372
    }
  },
  {
    "ph": "s", "id": 113, "pid": 1336754, "tid": 1336754, "ts": 1514656496327.052,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496366.096, "dur": 220.714,
    "args": {
      "External id": 2480670,"Sequence number": 24740536, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "9.9999999999999995e-07", "True", "False", "True"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [1], [], [8388608, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 2048], [2048], [], [16, 4096, 2048], [], [], [], []], "Ev Idx": 8373
    }
  },
  {
    "ph": "s", "id": 112, "pid": 1336754, "tid": 1336754, "ts": 1514656496366.096,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496386.018, "dur": 2.643,
    "args": {
      "External id": 2480671,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8374
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496386.703, "dur": 1.573,
    "args": {
      "External id": 2480672,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8375
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape_as", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496392.027, "dur": 4.232,
    "args": {
      "External id": 2480673,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [2048, 1]], "Input Dims": [[16, 4096, 2048], [65536, 2048]], "Ev Idx": 8376
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496393.134, "dur": 3.018,
    "args": {
      "External id": 2480674,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8377
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496395.114, "dur": 0.898,
    "args": {
      "External id": 2480675,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8378
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496403.433, "dur": 5.600,
    "args": {
      "External id": 2480676,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 8379
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496404.946, "dur": 3.727,
    "args": {
      "External id": 2480677,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8380
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496414.907, "dur": 3.766,
    "args": {
      "External id": 2480678,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8381
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496421.957, "dur": 3.266,
    "args": {
      "External id": 2480679,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8382
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496562.335, "dur": 3.847,
    "args": {
      "External id": 2480680,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8383
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496563.350, "dur": 2.542,
    "args": {
      "External id": 2480681,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8384
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496570.430, "dur": 2.120,
    "args": {
      "External id": 2480682,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8385
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496571.173, "dur": 1.222,
    "args": {
      "External id": 2480683,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8386
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496605.691, "dur": 95.572,
    "args": {
      "External id": 2480684,"Sequence number": 24740537, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [5632, 2048], []], "Ev Idx": 8387
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496606.677, "dur": 7.199,
    "args": {
      "External id": 2480685,"Sequence number": 24740537, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 8388
    }
  },
  {
    "ph": "s", "id": 111, "pid": 1336754, "tid": 1336754, "ts": 1514656496606.677,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496608.929, "dur": 3.886,
    "args": {
      "External id": 2480686,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 8389
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496610.930, "dur": 1.374,
    "args": {
      "External id": 2480687,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 8390
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496616.310, "dur": 84.584,
    "args": {
      "External id": 2480688,"Sequence number": 24740538, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 5632]], "Ev Idx": 8391
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496617.999, "dur": 3.506,
    "args": {
      "External id": 2480689,"Sequence number": 24740538, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8392
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496618.524, "dur": 2.864,
    "args": {
      "External id": 2480690,"Sequence number": 24740538, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8393
    }
  },
  {
    "ph": "s", "id": 110, "pid": 1336754, "tid": 1336754, "ts": 1514656496618.524,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496622.106, "dur": 71.981,
    "args": {
      "External id": 2480691,"Sequence number": 24740539, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 8394
    }
  },
  {
    "ph": "s", "id": 109, "pid": 1336754, "tid": 1336754, "ts": 1514656496622.106,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496696.446, "dur": 3.884,
    "args": {
      "External id": 2480692,"Sequence number": 24740540, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 8395
    }
  },
  {
    "ph": "s", "id": 108, "pid": 1336754, "tid": 1336754, "ts": 1514656496696.446,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496709.790, "dur": 66.816,
    "args": {
      "External id": 2480693,"Sequence number": 24740541, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [5632, 2048], []], "Ev Idx": 8396
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496710.331, "dur": 5.351,
    "args": {
      "External id": 2480694,"Sequence number": 24740541, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 8397
    }
  },
  {
    "ph": "s", "id": 107, "pid": 1336754, "tid": 1336754, "ts": 1514656496710.331,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496711.842, "dur": 2.776,
    "args": {
      "External id": 2480695,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 8398
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496713.399, "dur": 1.044,
    "args": {
      "External id": 2480696,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 8399
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496716.254, "dur": 60.049,
    "args": {
      "External id": 2480697,"Sequence number": 24740542, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 5632]], "Ev Idx": 8400
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496718.800, "dur": 5.499,
    "args": {
      "External id": 2480698,"Sequence number": 24740542, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8401
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496719.450, "dur": 4.667,
    "args": {
      "External id": 2480699,"Sequence number": 24740542, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8402
    }
  },
  {
    "ph": "s", "id": 106, "pid": 1336754, "tid": 1336754, "ts": 1514656496719.450,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496724.827, "dur": 45.784,
    "args": {
      "External id": 2480700,"Sequence number": 24740543, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 8403
    }
  },
  {
    "ph": "s", "id": 105, "pid": 1336754, "tid": 1336754, "ts": 1514656496724.827,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496771.932, "dur": 4.060,
    "args": {
      "External id": 2480701,"Sequence number": 24740544, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 8404
    }
  },
  {
    "ph": "s", "id": 104, "pid": 1336754, "tid": 1336754, "ts": 1514656496771.932,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SwiGLULinearFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496799.659, "dur": 136.182,
    "args": {
      "External id": 2480702,"Sequence number": 24740545, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[23068672, 5632, 1], [23068672, 5632, 1], [5632, 1], []], "Input Dims": [[16, 4096, 5632], [16, 4096, 5632], [2048, 5632], []], "Ev Idx": 8405
    }
  },
  {
    "ph": "s", "id": 103, "pid": 1336754, "tid": 1336754, "ts": 1514656496799.659,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496834.874, "dur": 4.300,
    "args": {
      "External id": 2480703,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8406
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496868.442, "dur": 56.039,
    "args": {
      "External id": 2480704,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[23068672, 5632, 1], [5632, 1], []], "Input Dims": [[16, 4096, 5632], [2048, 5632], []], "Ev Idx": 8407
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496868.976, "dur": 4.844,
    "args": {
      "External id": 2480705,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 8408
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496870.071, "dur": 2.866,
    "args": {
      "External id": 2480706,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 8409
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496871.481, "dur": 1.176,
    "args": {
      "External id": 2480707,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 8410
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496876.195, "dur": 47.919,
    "args": {
      "External id": 2480708,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[23068672, 5632, 1], [1, 5632]], "Input Dims": [[16, 4096, 5632], [5632, 2048]], "Ev Idx": 8411
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496877.442, "dur": 1.850,
    "args": {
      "External id": 2480709,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 8412
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496878.060, "dur": 1.086,
    "args": {
      "External id": 2480710,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 8413
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496879.806, "dur": 40.788,
    "args": {
      "External id": 2480711,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632]], "Input Dims": [[65536, 5632], [5632, 2048]], "Ev Idx": 8414
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496922.458, "dur": 1.068,
    "args": {
      "External id": 2480712,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8415
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336754, "tid": 1336754,
    "ts": 1514656496943.452, "dur": 23.486,
    "args": {
      "External id": 2480713,"Sequence number": 24740546, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 8416
    }
  },
  {
    "ph": "s", "id": 102, "pid": 1336754, "tid": 1336754, "ts": 1514656496943.452,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656497040.540, "dur": 180.499,
    "args": {
      "External id": 2480714,"Sequence number": 24740547, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "9.9999999999999995e-07", "False", "False", "True"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [1], [], [], [], [], [], []], "Input Dims": [[16, 4096, 2048], [2048], [], [], [], [], [], []], "Ev Idx": 8417
    }
  },
  {
    "ph": "s", "id": 101, "pid": 1336754, "tid": 1336754, "ts": 1514656497040.540,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656497061.645, "dur": 3.881,
    "args": {
      "External id": 2480715,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8418
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656497062.781, "dur": 2.444,
    "args": {
      "External id": 2480716,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8419
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656497073.625, "dur": 7.427,
    "args": {
      "External id": 2480717,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 8420
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656497075.945, "dur": 4.632,
    "args": {
      "External id": 2480718,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8421
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656497087.023, "dur": 3.537,
    "args": {
      "External id": 2480719,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8422
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656497204.678, "dur": 2.977,
    "args": {
      "External id": 2480720,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8423
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656497205.503, "dur": 1.960,
    "args": {
      "External id": 2480721,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8424
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656497239.875, "dur": 85.898,
    "args": {
      "External id": 2480722,"Sequence number": 24740548, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8425
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656497240.904, "dur": 6.522,
    "args": {
      "External id": 2480723,"Sequence number": 24740548, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8426
    }
  },
  {
    "ph": "s", "id": 100, "pid": 1336754, "tid": 1336754, "ts": 1514656497240.904,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656497243.051, "dur": 3.170,
    "args": {
      "External id": 2480724,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8427
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656497244.846, "dur": 1.158,
    "args": {
      "External id": 2480725,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8428
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656497248.145, "dur": 77.279,
    "args": {
      "External id": 2480726,"Sequence number": 24740549, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8429
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656497251.364, "dur": 3.459,
    "args": {
      "External id": 2480727,"Sequence number": 24740549, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8430
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656497252.036, "dur": 2.653,
    "args": {
      "External id": 2480728,"Sequence number": 24740549, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8431
    }
  },
  {
    "ph": "s", "id": 99, "pid": 1336754, "tid": 1336754, "ts": 1514656497252.036,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656497255.499, "dur": 62.511,
    "args": {
      "External id": 2480729,"Sequence number": 24740550, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8432
    }
  },
  {
    "ph": "s", "id": 98, "pid": 1336754, "tid": 1336754, "ts": 1514656497255.499,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656497320.155, "dur": 4.603,
    "args": {
      "External id": 2480730,"Sequence number": 24740551, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8433
    }
  },
  {
    "ph": "s", "id": 97, "pid": 1336754, "tid": 1336754, "ts": 1514656497320.155,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656497334.284, "dur": 58.977,
    "args": {
      "External id": 2480731,"Sequence number": 24740552, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8434
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656497336.432, "dur": 4.727,
    "args": {
      "External id": 2480732,"Sequence number": 24740552, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8435
    }
  },
  {
    "ph": "s", "id": 96, "pid": 1336754, "tid": 1336754, "ts": 1514656497336.432,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656497337.929, "dur": 2.072,
    "args": {
      "External id": 2480733,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8436
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656497339.166, "dur": 0.693,
    "args": {
      "External id": 2480734,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8437
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656497341.704, "dur": 51.333,
    "args": {
      "External id": 2480735,"Sequence number": 24740553, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8438
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656497342.667, "dur": 4.147,
    "args": {
      "External id": 2480736,"Sequence number": 24740553, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8439
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656497344.812, "dur": 1.867,
    "args": {
      "External id": 2480737,"Sequence number": 24740553, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8440
    }
  },
  {
    "ph": "s", "id": 95, "pid": 1336754, "tid": 1336754, "ts": 1514656497344.812,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656497347.368, "dur": 41.784,
    "args": {
      "External id": 2480738,"Sequence number": 24740554, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8441
    }
  },
  {
    "ph": "s", "id": 94, "pid": 1336754, "tid": 1336754, "ts": 1514656497347.368,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656497390.698, "dur": 2.021,
    "args": {
      "External id": 2480739,"Sequence number": 24740555, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8442
    }
  },
  {
    "ph": "s", "id": 93, "pid": 1336754, "tid": 1336754, "ts": 1514656497390.698,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656497400.465, "dur": 58.588,
    "args": {
      "External id": 2480740,"Sequence number": 24740556, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8443
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656497400.949, "dur": 7.153,
    "args": {
      "External id": 2480741,"Sequence number": 24740556, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8444
    }
  },
  {
    "ph": "s", "id": 92, "pid": 1336754, "tid": 1336754, "ts": 1514656497400.949,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656497403.201, "dur": 3.977,
    "args": {
      "External id": 2480742,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8445
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656497404.629, "dur": 2.358,
    "args": {
      "External id": 2480743,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8446
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656497408.676, "dur": 50.139,
    "args": {
      "External id": 2480744,"Sequence number": 24740557, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8447
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656497409.481, "dur": 5.752,
    "args": {
      "External id": 2480745,"Sequence number": 24740557, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8448
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656497412.337, "dur": 2.763,
    "args": {
      "External id": 2480746,"Sequence number": 24740557, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8449
    }
  },
  {
    "ph": "s", "id": 91, "pid": 1336754, "tid": 1336754, "ts": 1514656497412.337,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656497416.000, "dur": 37.098,
    "args": {
      "External id": 2480747,"Sequence number": 24740558, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8450
    }
  },
  {
    "ph": "s", "id": 90, "pid": 1336754, "tid": 1336754, "ts": 1514656497416.000,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656497454.600, "dur": 3.873,
    "args": {
      "External id": 2480748,"Sequence number": 24740559, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8451
    }
  },
  {
    "ph": "s", "id": 89, "pid": 1336754, "tid": 1336754, "ts": 1514656497454.600,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656497472.499, "dur": 4.511,
    "args": {
      "External id": 2480749,"Sequence number": 24740560, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8452
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656497473.258, "dur": 3.614,
    "args": {
      "External id": 2480750,"Sequence number": 24740560, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8453
    }
  },
  {
    "ph": "s", "id": 88, "pid": 1336754, "tid": 1336754, "ts": 1514656497473.258,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656497482.940, "dur": 2.732,
    "args": {
      "External id": 2480751,"Sequence number": 24740561, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8454
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656497483.898, "dur": 1.645,
    "args": {
      "External id": 2480752,"Sequence number": 24740561, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8455
    }
  },
  {
    "ph": "s", "id": 87, "pid": 1336754, "tid": 1336754, "ts": 1514656497483.898,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656497489.215, "dur": 2.972,
    "args": {
      "External id": 2480753,"Sequence number": 24740562, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8456
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656497490.207, "dur": 1.856,
    "args": {
      "External id": 2480754,"Sequence number": 24740562, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8457
    }
  },
  {
    "ph": "s", "id": 86, "pid": 1336754, "tid": 1336754, "ts": 1514656497490.207,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656497515.892, "dur": 162.253,
    "args": {
      "External id": 2480755,"Sequence number": 24740563, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "False", "False", "0", "", "8192"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [], [], [], [], []], "Ev Idx": 8458
    }
  },
  {
    "ph": "s", "id": 85, "pid": 1336754, "tid": 1336754, "ts": 1514656497515.892,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656497552.135, "dur": 9.528,
    "args": {
      "External id": 2480756,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8459
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656497554.665, "dur": 6.345,
    "args": {
      "External id": 2480757,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8460
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656497689.722, "dur": 96.691,
    "args": {
      "External id": 2480758,"Sequence number": 24740564, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "False", "False", "0", "", "8192"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [], [], [], [], []], "Ev Idx": 8461
    }
  },
  {
    "ph": "s", "id": 84, "pid": 1336754, "tid": 1336754, "ts": 1514656497689.722,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656497703.461, "dur": 5.940,
    "args": {
      "External id": 2480759,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8462
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656497705.155, "dur": 3.866,
    "args": {
      "External id": 2480760,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8463
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FlashAttnFunc", "pid": 1336754, "tid": 1336754,
    "ts": 1514656497813.714, "dur": 165.504,
    "args": {
      "External id": 2480761,"Sequence number": 24740565, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "", "True", "", "0.", "", "False", "False", "True"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "", "Scalar", "", "Scalar", "", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], []], "Ev Idx": 8464
    }
  },
  {
    "ph": "s", "id": 83, "pid": 1336754, "tid": 1336754, "ts": 1514656497813.714,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336754, "tid": 1336754,
    "ts": 1514656497837.982, "dur": 117.180,
    "args": {
      "External id": 2480762,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 8465
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656497881.399, "dur": 6.620,
    "args": {
      "External id": 2480763,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8466
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656497883.471, "dur": 4.124,
    "args": {
      "External id": 2480764,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8467
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656497890.378, "dur": 3.866,
    "args": {
      "External id": 2480765,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8468
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656497895.189, "dur": 1.561,
    "args": {
      "External id": 2480766,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8469
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656497899.282, "dur": 2.907,
    "args": {
      "External id": 2480767,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8470
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336754, "tid": 1336754,
    "ts": 1514656497966.089, "dur": 4.713,
    "args": {
      "External id": 2480768,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 8471
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498020.973, "dur": 8.457,
    "args": {
      "External id": 2480769,"Sequence number": 24740566, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 8472
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498023.579, "dur": 5.543,
    "args": {
      "External id": 2480770,"Sequence number": 24740566, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 8473
    }
  },
  {
    "ph": "s", "id": 82, "pid": 1336754, "tid": 1336754, "ts": 1514656498023.579,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498042.916, "dur": 112.030,
    "args": {
      "External id": 2480771,"Sequence number": 24740567, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8474
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498044.845, "dur": 10.698,
    "args": {
      "External id": 2480772,"Sequence number": 24740567, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8475
    }
  },
  {
    "ph": "s", "id": 81, "pid": 1336754, "tid": 1336754, "ts": 1514656498044.845,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498049.170, "dur": 5.391,
    "args": {
      "External id": 2480773,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8476
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498052.614, "dur": 1.620,
    "args": {
      "External id": 2480774,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8477
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498056.695, "dur": 97.918,
    "args": {
      "External id": 2480775,"Sequence number": 24740568, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8478
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498058.502, "dur": 3.220,
    "args": {
      "External id": 2480776,"Sequence number": 24740568, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8479
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498059.004, "dur": 2.597,
    "args": {
      "External id": 2480777,"Sequence number": 24740568, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8480
    }
  },
  {
    "ph": "s", "id": 80, "pid": 1336754, "tid": 1336754, "ts": 1514656498059.004,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498062.531, "dur": 84.437,
    "args": {
      "External id": 2480778,"Sequence number": 24740569, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8481
    }
  },
  {
    "ph": "s", "id": 79, "pid": 1336754, "tid": 1336754, "ts": 1514656498062.531,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498149.552, "dur": 4.341,
    "args": {
      "External id": 2480779,"Sequence number": 24740570, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8482
    }
  },
  {
    "ph": "s", "id": 78, "pid": 1336754, "tid": 1336754, "ts": 1514656498149.552,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498189.820, "dur": 204.214,
    "args": {
      "External id": 2480780,"Sequence number": 24740571, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "9.9999999999999995e-07", "True", "False", "True"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [1], [], [8388608, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 2048], [2048], [], [16, 4096, 2048], [], [], [], []], "Ev Idx": 8483
    }
  },
  {
    "ph": "s", "id": 77, "pid": 1336754, "tid": 1336754, "ts": 1514656498189.820,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498208.333, "dur": 2.489,
    "args": {
      "External id": 2480781,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8484
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498209.072, "dur": 1.569,
    "args": {
      "External id": 2480782,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8485
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape_as", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498214.817, "dur": 4.267,
    "args": {
      "External id": 2480783,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [2048, 1]], "Input Dims": [[16, 4096, 2048], [65536, 2048]], "Ev Idx": 8486
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498217.183, "dur": 1.780,
    "args": {
      "External id": 2480784,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8487
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498217.981, "dur": 0.855,
    "args": {
      "External id": 2480785,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8488
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498226.484, "dur": 6.965,
    "args": {
      "External id": 2480786,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 8489
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498228.534, "dur": 4.639,
    "args": {
      "External id": 2480787,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8490
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498239.296, "dur": 3.232,
    "args": {
      "External id": 2480788,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8491
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498245.968, "dur": 3.441,
    "args": {
      "External id": 2480789,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8492
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498373.731, "dur": 2.897,
    "args": {
      "External id": 2480790,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8493
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498374.675, "dur": 1.776,
    "args": {
      "External id": 2480791,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8494
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498378.861, "dur": 2.223,
    "args": {
      "External id": 2480792,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8495
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498379.653, "dur": 1.309,
    "args": {
      "External id": 2480793,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8496
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498412.082, "dur": 101.319,
    "args": {
      "External id": 2480794,"Sequence number": 24740572, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [5632, 2048], []], "Ev Idx": 8497
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498420.631, "dur": 8.004,
    "args": {
      "External id": 2480795,"Sequence number": 24740572, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 8498
    }
  },
  {
    "ph": "s", "id": 76, "pid": 1336754, "tid": 1336754, "ts": 1514656498420.631,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498424.120, "dur": 3.233,
    "args": {
      "External id": 2480796,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 8499
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498425.816, "dur": 1.299,
    "args": {
      "External id": 2480797,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 8500
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498429.499, "dur": 83.589,
    "args": {
      "External id": 2480798,"Sequence number": 24740573, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 5632]], "Ev Idx": 8501
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498430.683, "dur": 5.353,
    "args": {
      "External id": 2480799,"Sequence number": 24740573, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8502
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498431.305, "dur": 4.591,
    "args": {
      "External id": 2480800,"Sequence number": 24740573, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8503
    }
  },
  {
    "ph": "s", "id": 75, "pid": 1336754, "tid": 1336754, "ts": 1514656498431.305,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498436.640, "dur": 68.430,
    "args": {
      "External id": 2480801,"Sequence number": 24740574, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 8504
    }
  },
  {
    "ph": "s", "id": 74, "pid": 1336754, "tid": 1336754, "ts": 1514656498436.640,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498506.912, "dur": 5.575,
    "args": {
      "External id": 2480802,"Sequence number": 24740575, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 8505
    }
  },
  {
    "ph": "s", "id": 73, "pid": 1336754, "tid": 1336754, "ts": 1514656498506.912,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498521.782, "dur": 89.922,
    "args": {
      "External id": 2480803,"Sequence number": 24740576, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [5632, 2048], []], "Ev Idx": 8506
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498522.382, "dur": 24.562,
    "args": {
      "External id": 2480804,"Sequence number": 24740576, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 8507
    }
  },
  {
    "ph": "s", "id": 72, "pid": 1336754, "tid": 1336754, "ts": 1514656498522.382,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498523.809, "dur": 21.531,
    "args": {
      "External id": 2480805,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 8508
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498543.324, "dur": 1.525,
    "args": {
      "External id": 2480806,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 8509
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498548.028, "dur": 63.375,
    "args": {
      "External id": 2480807,"Sequence number": 24740577, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 5632]], "Ev Idx": 8510
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498549.500, "dur": 4.347,
    "args": {
      "External id": 2480808,"Sequence number": 24740577, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8511
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498550.178, "dur": 3.527,
    "args": {
      "External id": 2480809,"Sequence number": 24740577, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8512
    }
  },
  {
    "ph": "s", "id": 71, "pid": 1336754, "tid": 1336754, "ts": 1514656498550.178,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498556.049, "dur": 51.714,
    "args": {
      "External id": 2480810,"Sequence number": 24740578, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 8513
    }
  },
  {
    "ph": "s", "id": 70, "pid": 1336754, "tid": 1336754, "ts": 1514656498556.049,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498609.278, "dur": 1.671,
    "args": {
      "External id": 2480811,"Sequence number": 24740579, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 8514
    }
  },
  {
    "ph": "s", "id": 69, "pid": 1336754, "tid": 1336754, "ts": 1514656498609.278,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SwiGLULinearFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498634.039, "dur": 142.007,
    "args": {
      "External id": 2480812,"Sequence number": 24740580, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[23068672, 5632, 1], [23068672, 5632, 1], [5632, 1], []], "Input Dims": [[16, 4096, 5632], [16, 4096, 5632], [2048, 5632], []], "Ev Idx": 8515
    }
  },
  {
    "ph": "s", "id": 68, "pid": 1336754, "tid": 1336754, "ts": 1514656498634.039,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498670.530, "dur": 4.960,
    "args": {
      "External id": 2480813,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8516
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498705.766, "dur": 57.497,
    "args": {
      "External id": 2480814,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[23068672, 5632, 1], [5632, 1], []], "Input Dims": [[16, 4096, 5632], [2048, 5632], []], "Ev Idx": 8517
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498706.496, "dur": 4.650,
    "args": {
      "External id": 2480815,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 8518
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498707.542, "dur": 2.799,
    "args": {
      "External id": 2480816,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 8519
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498709.472, "dur": 0.640,
    "args": {
      "External id": 2480817,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 8520
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498711.828, "dur": 51.164,
    "args": {
      "External id": 2480818,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[23068672, 5632, 1], [1, 5632]], "Input Dims": [[16, 4096, 5632], [5632, 2048]], "Ev Idx": 8521
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498713.001, "dur": 2.117,
    "args": {
      "External id": 2480819,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 8522
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498713.793, "dur": 1.202,
    "args": {
      "External id": 2480820,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 8523
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498717.371, "dur": 41.994,
    "args": {
      "External id": 2480821,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632]], "Input Dims": [[65536, 5632], [5632, 2048]], "Ev Idx": 8524
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498761.274, "dur": 1.145,
    "args": {
      "External id": 2480822,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8525
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498784.018, "dur": 23.965,
    "args": {
      "External id": 2480823,"Sequence number": 24740581, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 8526
    }
  },
  {
    "ph": "s", "id": 67, "pid": 1336754, "tid": 1336754, "ts": 1514656498784.018,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498840.538, "dur": 244.374,
    "args": {
      "External id": 2480824,"Sequence number": 24740582, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "9.9999999999999995e-07", "False", "False", "True"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [1], [], [], [], [], [], []], "Input Dims": [[16, 4096, 2048], [2048], [], [], [], [], [], []], "Ev Idx": 8527
    }
  },
  {
    "ph": "s", "id": 66, "pid": 1336754, "tid": 1336754, "ts": 1514656498840.538,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498857.889, "dur": 2.620,
    "args": {
      "External id": 2480825,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8528
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498858.754, "dur": 1.552,
    "args": {
      "External id": 2480826,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8529
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498871.275, "dur": 12.876,
    "args": {
      "External id": 2480827,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 8530
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498879.649, "dur": 3.882,
    "args": {
      "External id": 2480828,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8531
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656498893.242, "dur": 4.924,
    "args": {
      "External id": 2480829,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8532
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499067.836, "dur": 4.877,
    "args": {
      "External id": 2480830,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8533
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499068.982, "dur": 3.384,
    "args": {
      "External id": 2480831,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8534
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499104.603, "dur": 110.584,
    "args": {
      "External id": 2480832,"Sequence number": 24740583, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8535
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499105.563, "dur": 8.276,
    "args": {
      "External id": 2480833,"Sequence number": 24740583, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8536
    }
  },
  {
    "ph": "s", "id": 65, "pid": 1336754, "tid": 1336754, "ts": 1514656499105.563,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499107.972, "dur": 4.667,
    "args": {
      "External id": 2480834,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8537
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499111.056, "dur": 1.331,
    "args": {
      "External id": 2480835,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8538
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499114.708, "dur": 100.182,
    "args": {
      "External id": 2480836,"Sequence number": 24740584, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8539
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499116.238, "dur": 3.370,
    "args": {
      "External id": 2480837,"Sequence number": 24740584, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8540
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499116.928, "dur": 2.564,
    "args": {
      "External id": 2480838,"Sequence number": 24740584, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8541
    }
  },
  {
    "ph": "s", "id": 64, "pid": 1336754, "tid": 1336754, "ts": 1514656499116.928,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499122.059, "dur": 87.343,
    "args": {
      "External id": 2480839,"Sequence number": 24740585, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8542
    }
  },
  {
    "ph": "s", "id": 63, "pid": 1336754, "tid": 1336754, "ts": 1514656499122.059,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499211.334, "dur": 2.847,
    "args": {
      "External id": 2480840,"Sequence number": 24740586, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8543
    }
  },
  {
    "ph": "s", "id": 62, "pid": 1336754, "tid": 1336754, "ts": 1514656499211.334,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499223.696, "dur": 80.241,
    "args": {
      "External id": 2480841,"Sequence number": 24740587, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8544
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499224.189, "dur": 8.360,
    "args": {
      "External id": 2480842,"Sequence number": 24740587, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8545
    }
  },
  {
    "ph": "s", "id": 61, "pid": 1336754, "tid": 1336754, "ts": 1514656499224.189,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499225.331, "dur": 5.778,
    "args": {
      "External id": 2480843,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8546
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499228.598, "dur": 2.284,
    "args": {
      "External id": 2480844,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8547
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499233.269, "dur": 70.388,
    "args": {
      "External id": 2480845,"Sequence number": 24740588, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8548
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499233.976, "dur": 2.966,
    "args": {
      "External id": 2480846,"Sequence number": 24740588, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8549
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499234.775, "dur": 2.039,
    "args": {
      "External id": 2480847,"Sequence number": 24740588, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8550
    }
  },
  {
    "ph": "s", "id": 60, "pid": 1336754, "tid": 1336754, "ts": 1514656499234.775,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499237.420, "dur": 60.274,
    "args": {
      "External id": 2480848,"Sequence number": 24740589, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8551
    }
  },
  {
    "ph": "s", "id": 59, "pid": 1336754, "tid": 1336754, "ts": 1514656499237.420,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499299.113, "dur": 4.179,
    "args": {
      "External id": 2480849,"Sequence number": 24740590, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8552
    }
  },
  {
    "ph": "s", "id": 58, "pid": 1336754, "tid": 1336754, "ts": 1514656499299.113,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499310.865, "dur": 60.454,
    "args": {
      "External id": 2480850,"Sequence number": 24740591, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8553
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499311.562, "dur": 4.694,
    "args": {
      "External id": 2480851,"Sequence number": 24740591, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8554
    }
  },
  {
    "ph": "s", "id": 57, "pid": 1336754, "tid": 1336754, "ts": 1514656499311.562,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499312.950, "dur": 2.230,
    "args": {
      "External id": 2480852,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8555
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499314.075, "dur": 0.860,
    "args": {
      "External id": 2480853,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8556
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499318.472, "dur": 52.582,
    "args": {
      "External id": 2480854,"Sequence number": 24740592, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8557
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499319.465, "dur": 4.062,
    "args": {
      "External id": 2480855,"Sequence number": 24740592, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8558
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499320.234, "dur": 3.168,
    "args": {
      "External id": 2480856,"Sequence number": 24740592, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8559
    }
  },
  {
    "ph": "s", "id": 56, "pid": 1336754, "tid": 1336754, "ts": 1514656499320.234,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499323.970, "dur": 41.328,
    "args": {
      "External id": 2480857,"Sequence number": 24740593, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8560
    }
  },
  {
    "ph": "s", "id": 55, "pid": 1336754, "tid": 1336754, "ts": 1514656499323.970,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499366.861, "dur": 3.796,
    "args": {
      "External id": 2480858,"Sequence number": 24740594, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8561
    }
  },
  {
    "ph": "s", "id": 54, "pid": 1336754, "tid": 1336754, "ts": 1514656499366.861,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499386.732, "dur": 3.390,
    "args": {
      "External id": 2480859,"Sequence number": 24740595, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8562
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499387.460, "dur": 2.471,
    "args": {
      "External id": 2480860,"Sequence number": 24740595, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8563
    }
  },
  {
    "ph": "s", "id": 53, "pid": 1336754, "tid": 1336754, "ts": 1514656499387.460,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499396.861, "dur": 2.748,
    "args": {
      "External id": 2480861,"Sequence number": 24740596, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8564
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499397.471, "dur": 2.004,
    "args": {
      "External id": 2480862,"Sequence number": 24740596, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8565
    }
  },
  {
    "ph": "s", "id": 52, "pid": 1336754, "tid": 1336754, "ts": 1514656499397.471,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499405.665, "dur": 4.189,
    "args": {
      "External id": 2480863,"Sequence number": 24740597, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8566
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499406.384, "dur": 3.315,
    "args": {
      "External id": 2480864,"Sequence number": 24740597, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8567
    }
  },
  {
    "ph": "s", "id": 51, "pid": 1336754, "tid": 1336754, "ts": 1514656499406.384,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499437.225, "dur": 173.017,
    "args": {
      "External id": 2480865,"Sequence number": 24740598, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "False", "False", "0", "", "8192"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [], [], [], [], []], "Ev Idx": 8568
    }
  },
  {
    "ph": "s", "id": 50, "pid": 1336754, "tid": 1336754, "ts": 1514656499437.225,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499456.061, "dur": 7.220,
    "args": {
      "External id": 2480866,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8569
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499458.331, "dur": 4.476,
    "args": {
      "External id": 2480867,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8570
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499624.418, "dur": 109.294,
    "args": {
      "External id": 2480868,"Sequence number": 24740599, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "False", "False", "0", "", "8192"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [], [], [], [], []], "Ev Idx": 8571
    }
  },
  {
    "ph": "s", "id": 49, "pid": 1336754, "tid": 1336754, "ts": 1514656499624.418,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499640.520, "dur": 7.628,
    "args": {
      "External id": 2480869,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8572
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499642.625, "dur": 4.988,
    "args": {
      "External id": 2480870,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8573
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FlashAttnFunc", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499759.361, "dur": 163.178,
    "args": {
      "External id": 2480871,"Sequence number": 24740600, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "", "True", "", "0.", "", "False", "False", "True"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "", "Scalar", "", "Scalar", "", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], []], "Ev Idx": 8574
    }
  },
  {
    "ph": "s", "id": 48, "pid": 1336754, "tid": 1336754, "ts": 1514656499759.361,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499785.472, "dur": 113.635,
    "args": {
      "External id": 2480872,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 8575
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499830.119, "dur": 6.312,
    "args": {
      "External id": 2480873,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8576
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499831.906, "dur": 4.148,
    "args": {
      "External id": 2480874,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8577
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499838.959, "dur": 4.011,
    "args": {
      "External id": 2480875,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8578
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499844.166, "dur": 2.809,
    "args": {
      "External id": 2480876,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8579
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499849.174, "dur": 3.229,
    "args": {
      "External id": 2480877,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8580
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499909.287, "dur": 4.210,
    "args": {
      "External id": 2480878,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 8581
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499927.338, "dur": 5.310,
    "args": {
      "External id": 2480879,"Sequence number": 24740601, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 8582
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499928.668, "dur": 3.804,
    "args": {
      "External id": 2480880,"Sequence number": 24740601, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 8583
    }
  },
  {
    "ph": "s", "id": 47, "pid": 1336754, "tid": 1336754, "ts": 1514656499928.668,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499943.318, "dur": 138.193,
    "args": {
      "External id": 2480881,"Sequence number": 24740602, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8584
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499946.026, "dur": 7.653,
    "args": {
      "External id": 2480882,"Sequence number": 24740602, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8585
    }
  },
  {
    "ph": "s", "id": 46, "pid": 1336754, "tid": 1336754, "ts": 1514656499946.026,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499948.332, "dur": 4.348,
    "args": {
      "External id": 2480883,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8586
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499950.829, "dur": 1.592,
    "args": {
      "External id": 2480884,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8587
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499954.716, "dur": 126.478,
    "args": {
      "External id": 2480885,"Sequence number": 24740603, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8588
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499956.340, "dur": 5.100,
    "args": {
      "External id": 2480886,"Sequence number": 24740603, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8589
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499958.836, "dur": 2.464,
    "args": {
      "External id": 2480887,"Sequence number": 24740603, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8590
    }
  },
  {
    "ph": "s", "id": 45, "pid": 1336754, "tid": 1336754, "ts": 1514656499958.836,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656499962.267, "dur": 110.085,
    "args": {
      "External id": 2480888,"Sequence number": 24740604, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8591
    }
  },
  {
    "ph": "s", "id": 44, "pid": 1336754, "tid": 1336754, "ts": 1514656499962.267,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656500075.976, "dur": 4.442,
    "args": {
      "External id": 2480889,"Sequence number": 24740605, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8592
    }
  },
  {
    "ph": "s", "id": 43, "pid": 1336754, "tid": 1336754, "ts": 1514656500075.976,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656500116.388, "dur": 196.704,
    "args": {
      "External id": 2480890,"Sequence number": 24740606, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "9.9999999999999995e-07", "True", "False", "True"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [1], [], [8388608, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 2048], [2048], [], [16, 4096, 2048], [], [], [], []], "Ev Idx": 8593
    }
  },
  {
    "ph": "s", "id": 42, "pid": 1336754, "tid": 1336754, "ts": 1514656500116.388,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656500136.765, "dur": 2.471,
    "args": {
      "External id": 2480891,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8594
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656500137.406, "dur": 1.629,
    "args": {
      "External id": 2480892,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8595
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape_as", "pid": 1336754, "tid": 1336754,
    "ts": 1514656500143.042, "dur": 3.416,
    "args": {
      "External id": 2480893,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [2048, 1]], "Input Dims": [[16, 4096, 2048], [65536, 2048]], "Ev Idx": 8596
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656500144.227, "dur": 2.106,
    "args": {
      "External id": 2480894,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8597
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656500145.121, "dur": 1.109,
    "args": {
      "External id": 2480895,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8598
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656500153.520, "dur": 9.134,
    "args": {
      "External id": 2480896,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 8599
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656500157.409, "dur": 4.943,
    "args": {
      "External id": 2480897,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8600
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656500168.407, "dur": 3.559,
    "args": {
      "External id": 2480898,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8601
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656500175.276, "dur": 3.837,
    "args": {
      "External id": 2480899,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8602
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656500293.022, "dur": 3.165,
    "args": {
      "External id": 2480900,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8603
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656500293.910, "dur": 1.994,
    "args": {
      "External id": 2480901,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8604
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656500298.494, "dur": 2.227,
    "args": {
      "External id": 2480902,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8605
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656500299.368, "dur": 1.252,
    "args": {
      "External id": 2480903,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8606
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656500331.682, "dur": 89.044,
    "args": {
      "External id": 2480904,"Sequence number": 24740607, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [5632, 2048], []], "Ev Idx": 8607
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656500332.806, "dur": 9.120,
    "args": {
      "External id": 2480905,"Sequence number": 24740607, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 8608
    }
  },
  {
    "ph": "s", "id": 41, "pid": 1336754, "tid": 1336754, "ts": 1514656500332.806,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656500335.402, "dur": 5.158,
    "args": {
      "External id": 2480906,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 8609
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656500337.311, "dur": 3.007,
    "args": {
      "External id": 2480907,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 8610
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656500342.804, "dur": 77.648,
    "args": {
      "External id": 2480908,"Sequence number": 24740608, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 5632]], "Ev Idx": 8611
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656500344.524, "dur": 4.973,
    "args": {
      "External id": 2480909,"Sequence number": 24740608, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8612
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656500346.792, "dur": 2.577,
    "args": {
      "External id": 2480910,"Sequence number": 24740608, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8613
    }
  },
  {
    "ph": "s", "id": 40, "pid": 1336754, "tid": 1336754, "ts": 1514656500346.792,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656500350.336, "dur": 62.885,
    "args": {
      "External id": 2480911,"Sequence number": 24740609, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 8614
    }
  },
  {
    "ph": "s", "id": 39, "pid": 1336754, "tid": 1336754, "ts": 1514656500350.336,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656500414.843, "dur": 5.099,
    "args": {
      "External id": 2480912,"Sequence number": 24740610, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 8615
    }
  },
  {
    "ph": "s", "id": 38, "pid": 1336754, "tid": 1336754, "ts": 1514656500414.843,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656500429.333, "dur": 66.466,
    "args": {
      "External id": 2480913,"Sequence number": 24740611, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [5632, 2048], []], "Ev Idx": 8616
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656500429.919, "dur": 6.896,
    "args": {
      "External id": 2480914,"Sequence number": 24740611, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 8617
    }
  },
  {
    "ph": "s", "id": 37, "pid": 1336754, "tid": 1336754, "ts": 1514656500429.919,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656500433.219, "dur": 2.508,
    "args": {
      "External id": 2480915,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 8618
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656500434.646, "dur": 0.915,
    "args": {
      "External id": 2480916,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 8619
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656500437.645, "dur": 57.965,
    "args": {
      "External id": 2480917,"Sequence number": 24740612, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 5632]], "Ev Idx": 8620
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656500438.723, "dur": 5.790,
    "args": {
      "External id": 2480918,"Sequence number": 24740612, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8621
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656500440.645, "dur": 3.740,
    "args": {
      "External id": 2480919,"Sequence number": 24740612, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8622
    }
  },
  {
    "ph": "s", "id": 36, "pid": 1336754, "tid": 1336754, "ts": 1514656500440.645,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656500445.127, "dur": 45.849,
    "args": {
      "External id": 2480920,"Sequence number": 24740613, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 8623
    }
  },
  {
    "ph": "s", "id": 35, "pid": 1336754, "tid": 1336754, "ts": 1514656500445.127,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656500492.703, "dur": 2.570,
    "args": {
      "External id": 2480921,"Sequence number": 24740614, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 8624
    }
  },
  {
    "ph": "s", "id": 34, "pid": 1336754, "tid": 1336754, "ts": 1514656500492.703,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SwiGLULinearFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656500515.720, "dur": 162.652,
    "args": {
      "External id": 2480922,"Sequence number": 24740615, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[23068672, 5632, 1], [23068672, 5632, 1], [5632, 1], []], "Input Dims": [[16, 4096, 5632], [16, 4096, 5632], [2048, 5632], []], "Ev Idx": 8625
    }
  },
  {
    "ph": "s", "id": 33, "pid": 1336754, "tid": 1336754, "ts": 1514656500515.720,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656500569.751, "dur": 4.934,
    "args": {
      "External id": 2480923,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8626
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656500605.432, "dur": 61.425,
    "args": {
      "External id": 2480924,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[23068672, 5632, 1], [5632, 1], []], "Input Dims": [[16, 4096, 5632], [2048, 5632], []], "Ev Idx": 8627
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656500606.346, "dur": 6.208,
    "args": {
      "External id": 2480925,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 8628
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656500607.540, "dur": 3.951,
    "args": {
      "External id": 2480926,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 8629
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656500609.127, "dur": 2.136,
    "args": {
      "External id": 2480927,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 8630
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656500613.172, "dur": 53.394,
    "args": {
      "External id": 2480928,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[23068672, 5632, 1], [1, 5632]], "Input Dims": [[16, 4096, 5632], [5632, 2048]], "Ev Idx": 8631
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336754, "tid": 1336754,
    "ts": 1514656500614.565, "dur": 4.015,
    "args": {
      "External id": 2480929,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 8632
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656500616.935, "dur": 1.530,
    "args": {
      "External id": 2480930,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 8633
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656500619.040, "dur": 43.747,
    "args": {
      "External id": 2480931,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632]], "Input Dims": [[65536, 5632], [5632, 2048]], "Ev Idx": 8634
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656500664.774, "dur": 1.091,
    "args": {
      "External id": 2480932,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8635
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336754, "tid": 1336754,
    "ts": 1514656500686.815, "dur": 23.334,
    "args": {
      "External id": 2480933,"Sequence number": 24740616, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 8636
    }
  },
  {
    "ph": "s", "id": 32, "pid": 1336754, "tid": 1336754, "ts": 1514656500686.815,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::stack", "pid": 1336754, "tid": 1336754,
    "ts": 1514656500728.443, "dur": 39.337,
    "args": {
      "External id": 2480934,"Sequence number": 24740617, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "-2"], "Input type": ["TensorList", "Scalar"], "Input Strides": [[[8388608, 2048, 1], [8388608, 2048, 1], [8388608, 2048, 1], [8388608, 2048, 1]], []], "Input Dims": [[[16, 4096, 2048], [16, 4096, 2048], [16, 4096, 2048], [16, 4096, 2048]], []], "Ev Idx": 8637
    }
  },
  {
    "ph": "s", "id": 31, "pid": 1336754, "tid": 1336754, "ts": 1514656500728.443,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::cat", "pid": 1336754, "tid": 1336754,
    "ts": 1514656500738.266, "dur": 25.292,
    "args": {
      "External id": 2480935,"Record function id": 0, "Concrete Inputs": ["", "2"], "Input type": ["TensorList", "Scalar"], "Input Strides": [[[8388608, 2048, 1], [8388608, 2048, 1], [8388608, 2048, 1], [8388608, 2048, 1]], []], "Input Dims": [[[16, 4096, 2048], [16, 4096, 2048], [16, 4096, 2048], [16, 4096, 2048]], []], "Ev Idx": 8638
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656500765.121, "dur": 1.158,
    "args": {
      "External id": 2480936,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 8192], []], "Ev Idx": 8639
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336754, "tid": 1336754,
    "ts": 1514656500800.036, "dur": 38.176,
    "args": {
      "External id": 2480937,"Record function id": 0, "Ev Idx": 8640
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 2/0", "pid": 1336754, "tid": 1336754,
    "ts": 1514656500839.023, "dur": 221.159,
    "args": {
      "External id": 2480938,"Record function id": 0, "Ev Idx": 8641
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656500875.748, "dur": 175.263,
    "args": {
      "External id": 2480939,"Sequence number": 24740618, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1], [33554432, 8192, 2048, 1]], "Input Dims": [[2048], [16, 4096, 4, 2048]], "Ev Idx": 8642
    }
  },
  {
    "ph": "s", "id": 30, "pid": 1336754, "tid": 1336754, "ts": 1514656500875.748,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336754, "tid": 1336754,
    "ts": 1514656500937.937, "dur": 34.226,
    "args": {
      "External id": 2480940,"kernel_hash": "caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/ao/caowdwy4dip5p7sg7hk44j6kom6frhip3u2qpmolzfoijzloazlo.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[262144, 2048], [262144, 2048], [2048], [262144], [], [], [], [], [], [], [], [], []], "Ev Idx": 8643
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501147.171, "dur": 39.105,
    "args": {
      "External id": 2480941,"Record function id": 0, "Concrete Inputs": ["[1]", "", "", "", "False"], "Input type": ["ScalarList", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 8644
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501149.733, "dur": 6.108,
    "args": {
      "External id": 2480942,"Record function id": 0, "Concrete Inputs": ["[1]", "", "", "", "False", ""], "Input type": ["ScalarList", "", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8645
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501159.031, "dur": 26.945,
    "args": {
      "External id": 2480943,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[1]], "Ev Idx": 8646
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501162.452, "dur": 22.869,
    "args": {
      "External id": 2480944,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[1], []], "Ev Idx": 8647
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501190.686, "dur": 17.692,
    "args": {
      "External id": 2480945,"Record function id": 0, "Concrete Inputs": ["[1]", "", "", "", "False"], "Input type": ["ScalarList", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 8648
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501191.515, "dur": 2.529,
    "args": {
      "External id": 2480946,"Record function id": 0, "Concrete Inputs": ["[1]", "", "", "", "False", ""], "Input type": ["ScalarList", "", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8649
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501196.166, "dur": 11.936,
    "args": {
      "External id": 2480947,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[1]], "Ev Idx": 8650
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501196.837, "dur": 10.740,
    "args": {
      "External id": 2480948,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[1], []], "Ev Idx": 8651
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501211.281, "dur": 13.943,
    "args": {
      "External id": 2480949,"Record function id": 0, "Concrete Inputs": ["[1]", "", "", "", "False"], "Input type": ["ScalarList", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 8652
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501211.829, "dur": 2.245,
    "args": {
      "External id": 2480950,"Record function id": 0, "Concrete Inputs": ["[1]", "", "", "", "False", ""], "Input type": ["ScalarList", "", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8653
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501214.660, "dur": 10.310,
    "args": {
      "External id": 2480951,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[1]], "Ev Idx": 8654
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501215.018, "dur": 9.589,
    "args": {
      "External id": 2480952,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[1], []], "Ev Idx": 8655
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501235.400, "dur": 0.614,
    "args": {
      "External id": 2480953,"Sequence number": 24740619, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "4", "0", "", "", "False", "False", ""], "Input type": ["long int", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[8192, 1], [], [], [], [], [], [], []], "Input Dims": [[16, 8192], [], [], [], [], [], [], []], "Ev Idx": 8656
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unfold", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501243.041, "dur": 10.314,
    "args": {
      "External id": 2480954,"Sequence number": 24740619, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "5", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1], [], [], []], "Input Dims": [[16, 8192], [], [], []], "Ev Idx": 8657
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501249.574, "dur": 2.005,
    "args": {
      "External id": 2480955,"Record function id": 0, "Concrete Inputs": ["", "[16, 8188, 5]", "[8192, 1, 1]", ""], "Input type": ["long int", "ScalarList", "ScalarList", ""], "Input Strides": [[8192, 1], [], [], []], "Input Dims": [[16, 8192], [], [], []], "Ev Idx": 8658
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501259.654, "dur": 6.308,
    "args": {
      "External id": 2480956,"Sequence number": 24740619, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], [], []], "Input Dims": [[16, 8188, 5], [], [], [], []], "Ev Idx": 8659
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501263.359, "dur": 0.785,
    "args": {
      "External id": 2480957,"Record function id": 0, "Concrete Inputs": ["", "[16, 8188, 5]", "[8192, 1, 1]", "0"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 8188, 5], [], [], []], "Ev Idx": 8660
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501267.107, "dur": 3.131,
    "args": {
      "External id": 2480958,"Sequence number": 24740619, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "0", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], [], []], "Input Dims": [[16, 8188, 5], [], [], [], []], "Ev Idx": 8661
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501268.673, "dur": 0.700,
    "args": {
      "External id": 2480959,"Record function id": 0, "Concrete Inputs": ["", "[16, 8188, 5]", "[8192, 1, 1]", "0"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 8188, 5], [], [], []], "Ev Idx": 8662
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501271.520, "dur": 2.966,
    "args": {
      "External id": 2480960,"Sequence number": 24740619, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "2", "1", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], [], []], "Input Dims": [[16, 8188, 5], [], [], [], []], "Ev Idx": 8663
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501273.242, "dur": 0.749,
    "args": {
      "External id": 2480961,"Record function id": 0, "Concrete Inputs": ["", "[16, 8188, 4]", "[8192, 1, 1]", "1"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 8188, 5], [], [], []], "Ev Idx": 8664
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501277.385, "dur": 2.985,
    "args": {
      "External id": 2480962,"Sequence number": 24740619, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], [], []], "Input Dims": [[16, 8188, 4], [], [], [], []], "Ev Idx": 8665
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501279.207, "dur": 0.622,
    "args": {
      "External id": 2480963,"Record function id": 0, "Concrete Inputs": ["", "[16, 8188, 4]", "[8192, 1, 1]", "1"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 8188, 4], [], [], []], "Ev Idx": 8666
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501281.465, "dur": 3.244,
    "args": {
      "External id": 2480964,"Sequence number": 24740619, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "0", "4096", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], [], []], "Input Dims": [[16, 8188, 4], [], [], [], []], "Ev Idx": 8667
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501283.078, "dur": 0.784,
    "args": {
      "External id": 2480965,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4]", "[8192, 1, 1]", "1"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 8188, 4], [], [], []], "Ev Idx": 8668
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501285.779, "dur": 2.667,
    "args": {
      "External id": 2480966,"Sequence number": 24740619, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "2", "0", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], [], []], "Input Dims": [[16, 4096, 4], [], [], [], []], "Ev Idx": 8669
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501287.233, "dur": 0.699,
    "args": {
      "External id": 2480967,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4]", "[8192, 1, 1]", "1"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 4096, 4], [], [], []], "Ev Idx": 8670
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501292.112, "dur": 4.347,
    "args": {
      "External id": 2480968,"Sequence number": 24740619, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "2"], "Input type": ["long int", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], []], "Input Dims": [[16, 4096, 4], [], []], "Ev Idx": 8671
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501295.228, "dur": 0.456,
    "args": {
      "External id": 2480969,"Record function id": 0, "Concrete Inputs": ["", "[16, 4, 4096]", "[8192, 1, 1]", ""], "Input type": ["long int", "ScalarList", "ScalarList", ""], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 4096, 4], [], [], []], "Ev Idx": 8672
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501300.512, "dur": 2.814,
    "args": {
      "External id": 2480970,"Sequence number": 24740619, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], [], []], "Input Dims": [[16, 4, 4096], [], [], [], []], "Ev Idx": 8673
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501302.102, "dur": 0.647,
    "args": {
      "External id": 2480971,"Record function id": 0, "Concrete Inputs": ["", "[16, 4, 4096]", "[8192, 1, 1]", "1"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 4, 4096], [], [], []], "Ev Idx": 8674
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501309.089, "dur": 6.875,
    "args": {
      "External id": 2480972,"Sequence number": 24740619, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "0"], "Input type": ["long int", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], []], "Input Dims": [[16, 4, 4096], [], []], "Ev Idx": 8675
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501314.196, "dur": 0.645,
    "args": {
      "External id": 2480973,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096]", "[8192, 1]", "1"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 4, 4096], [], [], []], "Ev Idx": 8676
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501316.982, "dur": 2.555,
    "args": {
      "External id": 2480974,"Sequence number": 24740619, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "0", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], []], "Ev Idx": 8677
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501318.444, "dur": 0.544,
    "args": {
      "External id": 2480975,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096]", "[8192, 1]", "1"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1], [], [], []], "Input Dims": [[16, 4096], [], [], []], "Ev Idx": 8678
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501322.049, "dur": 7.273,
    "args": {
      "External id": 2480976,"Sequence number": 24740619, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 8679
    }
  },
  {
    "ph": "s", "id": 29, "pid": 1336754, "tid": 1336754, "ts": 1514656501322.049,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501326.973, "dur": 0.954,
    "args": {
      "External id": 2480977,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 8680
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501330.310, "dur": 4.740,
    "args": {
      "External id": 2480978,"Sequence number": 24740620, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 8681
    }
  },
  {
    "ph": "s", "id": 28, "pid": 1336754, "tid": 1336754, "ts": 1514656501330.310,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501333.592, "dur": 0.739,
    "args": {
      "External id": 2480979,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 8682
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501335.983, "dur": 6.337,
    "args": {
      "External id": 2480980,"Sequence number": 24740621, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "2", "0"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], []], "Input Dims": [[16, 4096, 4, 2048], [], []], "Ev Idx": 8683
    }
  },
  {
    "ph": "s", "id": 27, "pid": 1336754, "tid": 1336754, "ts": 1514656501335.983,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501340.626, "dur": 0.823,
    "args": {
      "External id": 2480981,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 8684
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501343.359, "dur": 5.191,
    "args": {
      "External id": 2480982,"Sequence number": 24740622, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "2", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 1], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], []], "Ev Idx": 8685
    }
  },
  {
    "ph": "s", "id": 26, "pid": 1336754, "tid": 1336754, "ts": 1514656501343.359,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501345.674, "dur": 1.993,
    "args": {
      "External id": 2480983,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 1], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], []], "Ev Idx": 8686
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::contiguous", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501353.087, "dur": 38.693,
    "args": {
      "External id": 2480984,"Sequence number": 24740623, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["long int", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 8687
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501355.169, "dur": 36.297,
    "args": {
      "External id": 2480985,"Sequence number": 24740623, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["long int", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 8688
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501357.851, "dur": 5.505,
    "args": {
      "External id": 2480986,"Record function id": 0, "Concrete Inputs": ["", "4", "0", "", "", "0"], "Input type": ["long int", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[8192, 1], [], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], [], []], "Ev Idx": 8689
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501359.915, "dur": 2.858,
    "args": {
      "External id": 2480987,"Record function id": 0, "Concrete Inputs": ["[16, 4096]", "4", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8690
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501364.897, "dur": 26.000,
    "args": {
      "External id": 2480988,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[4096, 1], [8192, 1], []], "Input Dims": [[16, 4096], [16, 4096], []], "Ev Idx": 8691
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501412.407, "dur": 4.306,
    "args": {
      "External id": 2480989,"Sequence number": 24740623, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8692
    }
  },
  {
    "ph": "s", "id": 25, "pid": 1336754, "tid": 1336754, "ts": 1514656501412.407,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501418.644, "dur": 1.306,
    "args": {
      "External id": 2480990,"Sequence number": 24740624, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["long int", "ScalarList"], "Input Strides": [[4096, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 8693
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FusedLinearCrossEntropyFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501446.694, "dur": 72835.346,
    "args": {
      "External id": 2480991,"Sequence number": 24740624, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "-100", "0.", "1.", "8"], "Input type": ["c10::BFloat16", "long int", "c10::BFloat16", "", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1], [1], [2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536], [32000, 2048], [], [], [], [], []], "Ev Idx": 8694
    }
  },
  {
    "ph": "s", "id": 24, "pid": 1336754, "tid": 1336754, "ts": 1514656501446.694,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::contiguous", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501459.338, "dur": 26.538,
    "args": {
      "External id": 2480992,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8695
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501459.937, "dur": 25.739,
    "args": {
      "External id": 2480993,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8696
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501461.226, "dur": 6.063,
    "args": {
      "External id": 2480994,"Record function id": 0, "Concrete Inputs": ["", "15", "0", "", "", "0"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[8192, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 8697
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501462.438, "dur": 4.425,
    "args": {
      "External id": 2480995,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8698
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501468.139, "dur": 17.102,
    "args": {
      "External id": 2480996,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [8192, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 8699
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501503.032, "dur": 39.766,
    "args": {
      "External id": 2480997,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 8700
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501504.173, "dur": 5.872,
    "args": {
      "External id": 2480998,"Record function id": 0, "Concrete Inputs": ["", "15", "0", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 8701
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501506.210, "dur": 3.498,
    "args": {
      "External id": 2480999,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8702
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501511.228, "dur": 31.053,
    "args": {
      "External id": 2481000,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 8703
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501512.366, "dur": 14.033,
    "args": {
      "External id": 2481001,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8704
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501547.979, "dur": 30.511,
    "args": {
      "External id": 2481002,"Record function id": 0, "Concrete Inputs": ["", "6", "", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], []], "Ev Idx": 8705
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501548.800, "dur": 6.963,
    "args": {
      "External id": 2481003,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], []], "Ev Idx": 8706
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501552.034, "dur": 3.505,
    "args": {
      "External id": 2481004,"Record function id": 0, "Concrete Inputs": ["[32000, 2048]", "[2048, 1]", "6", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8707
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501564.372, "dur": 13.940,
    "args": {
      "External id": 2481005,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 8708
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501565.067, "dur": 12.827,
    "args": {
      "External id": 2481006,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[2048, 1], []], "Input Dims": [[32000, 2048], []], "Ev Idx": 8709
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501583.378, "dur": 21.725,
    "args": {
      "External id": 2481007,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False"], "Input type": ["ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 8710
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501589.694, "dur": 4.256,
    "args": {
      "External id": 2481008,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8711
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501594.504, "dur": 10.288,
    "args": {
      "External id": 2481009,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[65536]], "Ev Idx": 8712
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501595.144, "dur": 9.379,
    "args": {
      "External id": 2481010,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 8713
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::ne", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501614.183, "dur": 25.464,
    "args": {
      "External id": 2481011,"Record function id": 0, "Concrete Inputs": ["", "-100"], "Input type": ["long int", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 8714
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501644.074, "dur": 55.719,
    "args": {
      "External id": 2481012,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["bool", ""], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 8715
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501647.814, "dur": 51.539,
    "args": {
      "External id": 2481013,"Record function id": 0, "Concrete Inputs": ["", "[]", "False", ""], "Input type": ["bool", "ScalarList", "Scalar", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 8716
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501653.795, "dur": 0.838,
    "args": {
      "External id": 2481014,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[0]", ""], "Input type": ["long int", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 8717
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501655.940, "dur": 25.100,
    "args": {
      "External id": 2481015,"Record function id": 0, "Concrete Inputs": ["", "4", "False", "False", ""], "Input type": ["bool", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 8718
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501657.606, "dur": 23.200,
    "args": {
      "External id": 2481016,"Record function id": 0, "Concrete Inputs": ["", "4", "", "", "", "False", ""], "Input type": ["bool", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[65536], [], [], [], [], [], []], "Ev Idx": 8719
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501661.595, "dur": 3.071,
    "args": {
      "External id": 2481017,"Record function id": 0, "Concrete Inputs": ["[65536]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8720
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501665.472, "dur": 14.928,
    "args": {
      "External id": 2481018,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["long int", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[65536], [65536], []], "Ev Idx": 8721
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::item", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501705.776, "dur": 66914.597,
    "args": {
      "External id": 2481019,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["long int"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 8722
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_local_scalar_dense", "pid": 1336754, "tid": 1336754,
    "ts": 1514656501707.751, "dur": 66911.807,
    "args": {
      "External id": 2481020,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["long int"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 8723
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656568632.366, "dur": 8.461,
    "args": {
      "External id": 2481021,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 8724
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656568637.522, "dur": 1.334,
    "args": {
      "External id": 2481022,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 8725
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656568645.679, "dur": 106.769,
    "args": {
      "External id": 2481023,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 8726
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656568646.967, "dur": 10.415,
    "args": {
      "External id": 2481024,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 8727
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656568651.054, "dur": 5.531,
    "args": {
      "External id": 2481025,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 8728
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656568655.583, "dur": 0.746,
    "args": {
      "External id": 2481026,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 8729
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656568658.589, "dur": 93.050,
    "args": {
      "External id": 2481027,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 8730
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656568660.389, "dur": 90.247,
    "args": {
      "External id": 2481028,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 8731
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656568756.373, "dur": 4.375,
    "args": {
      "External id": 2481029,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 8732
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656568758.662, "dur": 0.688,
    "args": {
      "External id": 2481030,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "0"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 8733
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656568769.710, "dur": 3.631,
    "args": {
      "External id": 2481031,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 8734
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656568785.773, "dur": 7.315,
    "args": {
      "External id": 2481032,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 8735
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656568788.654, "dur": 4.143,
    "args": {
      "External id": 2481033,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8736
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656568932.586, "dur": 265.391,
    "args": {
      "External id": 2481034,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 8737
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656568937.791, "dur": 2.289,
    "args": {
      "External id": 2481035,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8738
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656568943.368, "dur": 254.063,
    "args": {
      "External id": 2481036,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 8739
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336754, "tid": 1336754,
    "ts": 1514656568947.305, "dur": 0.723,
    "args": {
      "External id": 2481037,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 8740
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336754, "tid": 1336754,
    "ts": 1514656568953.378, "dur": 53.901,
    "args": {
      "External id": 2481038,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 8741
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569011.636, "dur": 5.842,
    "args": {
      "External id": 2481039,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 8742
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569015.911, "dur": 1.155,
    "args": {
      "External id": 2481040,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 8743
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569020.151, "dur": 28.696,
    "args": {
      "External id": 2481041,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 8744
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569021.465, "dur": 2.264,
    "args": {
      "External id": 2481042,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8745
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569025.874, "dur": 22.701,
    "args": {
      "External id": 2481043,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 8746
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569030.500, "dur": 3.561,
    "args": {
      "External id": 2481044,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 8747
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569051.331, "dur": 23.164,
    "args": {
      "External id": 2481045,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 8748
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569077.053, "dur": 20.556,
    "args": {
      "External id": 2481046,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 8749
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569102.061, "dur": 16.590,
    "args": {
      "External id": 2481047,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 8750
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569124.036, "dur": 16.370,
    "args": {
      "External id": 2481048,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 8751
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569143.374, "dur": 21.481,
    "args": {
      "External id": 2481049,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 8752
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569145.843, "dur": 1.978,
    "args": {
      "External id": 2481050,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 8753
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569150.058, "dur": 0.719,
    "args": {
      "External id": 2481051,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 8754
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569168.706, "dur": 14.648,
    "args": {
      "External id": 2481052,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 8755
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569185.464, "dur": 10.791,
    "args": {
      "External id": 2481053,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 8756
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569208.099, "dur": 2.447,
    "args": {
      "External id": 2481054,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 8757
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569218.171, "dur": 4.202,
    "args": {
      "External id": 2481055,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 8758
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569220.437, "dur": 0.829,
    "args": {
      "External id": 2481056,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 8759
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569313.029, "dur": 71.166,
    "args": {
      "External id": 2481057,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 8760
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569389.700, "dur": 5.126,
    "args": {
      "External id": 2481058,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 8761
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569392.671, "dur": 0.969,
    "args": {
      "External id": 2481059,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 8762
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569396.364, "dur": 31.619,
    "args": {
      "External id": 2481060,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 8763
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569433.115, "dur": 7.579,
    "args": {
      "External id": 2481061,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 8764
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569436.294, "dur": 3.678,
    "args": {
      "External id": 2481062,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 8765
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569438.084, "dur": 1.625,
    "args": {
      "External id": 2481063,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 8766
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569445.055, "dur": 47.700,
    "args": {
      "External id": 2481064,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 8767
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569446.626, "dur": 45.551,
    "args": {
      "External id": 2481065,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 8768
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569497.918, "dur": 16.967,
    "args": {
      "External id": 2481066,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 8769
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569521.117, "dur": 5.587,
    "args": {
      "External id": 2481067,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 8770
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569525.049, "dur": 0.741,
    "args": {
      "External id": 2481068,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "16777216"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 8771
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569547.052, "dur": 53.842,
    "args": {
      "External id": 2481069,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 8772
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569548.112, "dur": 4.203,
    "args": {
      "External id": 2481070,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 8773
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569549.086, "dur": 2.530,
    "args": {
      "External id": 2481071,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 8774
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569550.571, "dur": 0.811,
    "args": {
      "External id": 2481072,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 8775
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569553.079, "dur": 47.413,
    "args": {
      "External id": 2481073,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 8776
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569553.833, "dur": 46.118,
    "args": {
      "External id": 2481074,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 8777
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569606.978, "dur": 4.341,
    "args": {
      "External id": 2481075,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 8778
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569609.194, "dur": 0.753,
    "args": {
      "External id": 2481076,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "8192"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 8779
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569617.992, "dur": 1.594,
    "args": {
      "External id": 2481077,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 8780
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569627.290, "dur": 7.508,
    "args": {
      "External id": 2481078,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 8781
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569629.090, "dur": 5.410,
    "args": {
      "External id": 2481079,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8782
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569735.856, "dur": 184.110,
    "args": {
      "External id": 2481080,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 8783
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569737.843, "dur": 2.213,
    "args": {
      "External id": 2481081,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8784
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569741.947, "dur": 177.561,
    "args": {
      "External id": 2481082,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 8785
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569743.308, "dur": 0.306,
    "args": {
      "External id": 2481083,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 8786
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569744.727, "dur": 24.149,
    "args": {
      "External id": 2481084,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 8787
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569770.621, "dur": 3.541,
    "args": {
      "External id": 2481085,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 8788
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569772.902, "dur": 0.822,
    "args": {
      "External id": 2481086,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 8789
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569775.119, "dur": 26.235,
    "args": {
      "External id": 2481087,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 8790
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569776.635, "dur": 1.824,
    "args": {
      "External id": 2481088,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8791
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569781.670, "dur": 19.416,
    "args": {
      "External id": 2481089,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 8792
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569786.491, "dur": 2.794,
    "args": {
      "External id": 2481090,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 8793
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569802.928, "dur": 21.711,
    "args": {
      "External id": 2481091,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 8794
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569826.404, "dur": 13.419,
    "args": {
      "External id": 2481092,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 8795
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569842.930, "dur": 14.429,
    "args": {
      "External id": 2481093,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 8796
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569858.907, "dur": 11.790,
    "args": {
      "External id": 2481094,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 8797
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569872.340, "dur": 18.411,
    "args": {
      "External id": 2481095,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 8798
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569874.456, "dur": 1.265,
    "args": {
      "External id": 2481096,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 8799
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569877.575, "dur": 0.721,
    "args": {
      "External id": 2481097,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 8800
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569894.433, "dur": 11.862,
    "args": {
      "External id": 2481098,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 8801
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569907.356, "dur": 11.176,
    "args": {
      "External id": 2481099,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 8802
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569926.850, "dur": 2.267,
    "args": {
      "External id": 2481100,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 8803
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569937.534, "dur": 3.230,
    "args": {
      "External id": 2481101,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 8804
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656569939.459, "dur": 0.545,
    "args": {
      "External id": 2481102,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "8192"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 8805
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570048.274, "dur": 59.853,
    "args": {
      "External id": 2481103,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 8806
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570113.770, "dur": 8.994,
    "args": {
      "External id": 2481104,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 8807
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570119.222, "dur": 1.993,
    "args": {
      "External id": 2481105,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "16777216"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 8808
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570124.164, "dur": 27.119,
    "args": {
      "External id": 2481106,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 8809
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570156.190, "dur": 5.176,
    "args": {
      "External id": 2481107,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 8810
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570157.637, "dur": 2.865,
    "args": {
      "External id": 2481108,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 8811
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570159.489, "dur": 0.848,
    "args": {
      "External id": 2481109,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 8812
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570164.035, "dur": 42.070,
    "args": {
      "External id": 2481110,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 8813
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570165.164, "dur": 40.400,
    "args": {
      "External id": 2481111,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 8814
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570211.830, "dur": 15.705,
    "args": {
      "External id": 2481112,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 8815
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570233.241, "dur": 3.863,
    "args": {
      "External id": 2481113,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 8816
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570235.492, "dur": 0.578,
    "args": {
      "External id": 2481114,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "33554432"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 8817
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570240.983, "dur": 51.237,
    "args": {
      "External id": 2481115,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 8818
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570241.962, "dur": 7.026,
    "args": {
      "External id": 2481116,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 8819
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570242.691, "dur": 5.685,
    "args": {
      "External id": 2481117,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 8820
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570246.134, "dur": 2.034,
    "args": {
      "External id": 2481118,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 8821
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570249.583, "dur": 42.160,
    "args": {
      "External id": 2481119,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 8822
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570250.175, "dur": 41.067,
    "args": {
      "External id": 2481120,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 8823
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570296.299, "dur": 3.729,
    "args": {
      "External id": 2481121,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 8824
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570298.181, "dur": 0.602,
    "args": {
      "External id": 2481122,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "16384"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 8825
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570305.781, "dur": 1.440,
    "args": {
      "External id": 2481123,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 8826
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570315.185, "dur": 7.959,
    "args": {
      "External id": 2481124,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 8827
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570318.942, "dur": 3.939,
    "args": {
      "External id": 2481125,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8828
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570413.608, "dur": 250.755,
    "args": {
      "External id": 2481126,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 8829
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570416.038, "dur": 2.032,
    "args": {
      "External id": 2481127,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8830
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570419.338, "dur": 244.467,
    "args": {
      "External id": 2481128,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 8831
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570420.427, "dur": 0.241,
    "args": {
      "External id": 2481129,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 8832
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570423.354, "dur": 19.740,
    "args": {
      "External id": 2481130,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 8833
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570444.958, "dur": 3.358,
    "args": {
      "External id": 2481131,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 8834
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570446.812, "dur": 1.221,
    "args": {
      "External id": 2481132,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 8835
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570449.070, "dur": 23.066,
    "args": {
      "External id": 2481133,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 8836
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570450.173, "dur": 2.802,
    "args": {
      "External id": 2481134,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8837
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570455.839, "dur": 16.040,
    "args": {
      "External id": 2481135,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 8838
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570458.249, "dur": 2.819,
    "args": {
      "External id": 2481136,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 8839
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570473.726, "dur": 19.657,
    "args": {
      "External id": 2481137,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 8840
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570495.035, "dur": 12.922,
    "args": {
      "External id": 2481138,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 8841
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570510.528, "dur": 13.329,
    "args": {
      "External id": 2481139,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 8842
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570525.258, "dur": 30.409,
    "args": {
      "External id": 2481140,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 8843
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570558.455, "dur": 26.586,
    "args": {
      "External id": 2481141,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 8844
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570560.319, "dur": 2.049,
    "args": {
      "External id": 2481142,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 8845
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570566.318, "dur": 0.710,
    "args": {
      "External id": 2481143,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 8846
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570586.712, "dur": 31.771,
    "args": {
      "External id": 2481144,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 8847
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570619.460, "dur": 43.231,
    "args": {
      "External id": 2481145,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 8848
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570672.048, "dur": 2.205,
    "args": {
      "External id": 2481146,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 8849
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570683.746, "dur": 3.441,
    "args": {
      "External id": 2481147,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 8850
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570685.957, "dur": 0.472,
    "args": {
      "External id": 2481148,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "16384"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 8851
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570754.536, "dur": 70.523,
    "args": {
      "External id": 2481149,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 8852
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570830.031, "dur": 6.482,
    "args": {
      "External id": 2481150,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 8853
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570834.328, "dur": 1.049,
    "args": {
      "External id": 2481151,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "33554432"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 8854
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570837.924, "dur": 27.244,
    "args": {
      "External id": 2481152,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 8855
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570901.835, "dur": 7.345,
    "args": {
      "External id": 2481153,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 8856
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570903.562, "dur": 4.964,
    "args": {
      "External id": 2481154,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 8857
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570905.929, "dur": 2.414,
    "args": {
      "External id": 2481155,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 8858
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570911.843, "dur": 44.813,
    "args": {
      "External id": 2481156,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 8859
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570914.830, "dur": 41.324,
    "args": {
      "External id": 2481157,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 8860
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570960.741, "dur": 13.862,
    "args": {
      "External id": 2481158,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 8861
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656570979.961, "dur": 38.365,
    "args": {
      "External id": 2481159,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 8862
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571015.571, "dur": 1.034,
    "args": {
      "External id": 2481160,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "50331648"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 8863
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571024.216, "dur": 55.410,
    "args": {
      "External id": 2481161,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 8864
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571024.956, "dur": 5.341,
    "args": {
      "External id": 2481162,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 8865
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571025.939, "dur": 3.771,
    "args": {
      "External id": 2481163,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 8866
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571028.976, "dur": 0.575,
    "args": {
      "External id": 2481164,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 8867
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571030.903, "dur": 48.252,
    "args": {
      "External id": 2481165,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 8868
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571031.583, "dur": 47.038,
    "args": {
      "External id": 2481166,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 8869
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571084.137, "dur": 4.048,
    "args": {
      "External id": 2481167,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 8870
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571086.255, "dur": 0.679,
    "args": {
      "External id": 2481168,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "24576"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 8871
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571094.766, "dur": 1.514,
    "args": {
      "External id": 2481169,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 8872
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571105.527, "dur": 6.391,
    "args": {
      "External id": 2481170,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 8873
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571107.318, "dur": 4.368,
    "args": {
      "External id": 2481171,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8874
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571200.255, "dur": 173.536,
    "args": {
      "External id": 2481172,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 8875
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571202.165, "dur": 2.032,
    "args": {
      "External id": 2481173,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8876
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571207.315, "dur": 165.905,
    "args": {
      "External id": 2481174,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 8877
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571208.410, "dur": 0.265,
    "args": {
      "External id": 2481175,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 8878
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571211.445, "dur": 21.281,
    "args": {
      "External id": 2481176,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 8879
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571234.286, "dur": 4.067,
    "args": {
      "External id": 2481177,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 8880
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571236.158, "dur": 1.891,
    "args": {
      "External id": 2481178,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 8881
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571239.145, "dur": 19.516,
    "args": {
      "External id": 2481179,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 8882
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571240.160, "dur": 1.284,
    "args": {
      "External id": 2481180,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8883
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571242.726, "dur": 15.690,
    "args": {
      "External id": 2481181,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 8884
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571245.228, "dur": 2.305,
    "args": {
      "External id": 2481182,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 8885
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571260.019, "dur": 20.099,
    "args": {
      "External id": 2481183,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 8886
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571281.627, "dur": 13.064,
    "args": {
      "External id": 2481184,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 8887
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571297.996, "dur": 13.649,
    "args": {
      "External id": 2481185,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 8888
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571314.758, "dur": 12.027,
    "args": {
      "External id": 2481186,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 8889
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571328.360, "dur": 18.959,
    "args": {
      "External id": 2481187,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 8890
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571330.185, "dur": 1.287,
    "args": {
      "External id": 2481188,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 8891
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571333.252, "dur": 1.926,
    "args": {
      "External id": 2481189,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 8892
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571349.060, "dur": 11.273,
    "args": {
      "External id": 2481190,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 8893
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571361.428, "dur": 10.927,
    "args": {
      "External id": 2481191,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 8894
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571380.210, "dur": 1.683,
    "args": {
      "External id": 2481192,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 8895
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571391.812, "dur": 3.132,
    "args": {
      "External id": 2481193,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 8896
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571393.672, "dur": 0.523,
    "args": {
      "External id": 2481194,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "24576"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 8897
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571454.802, "dur": 48.615,
    "args": {
      "External id": 2481195,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 8898
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571508.133, "dur": 4.412,
    "args": {
      "External id": 2481196,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 8899
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571510.448, "dur": 0.907,
    "args": {
      "External id": 2481197,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "50331648"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 8900
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571513.898, "dur": 43.586,
    "args": {
      "External id": 2481198,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 8901
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571563.521, "dur": 7.103,
    "args": {
      "External id": 2481199,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 8902
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571564.914, "dur": 4.923,
    "args": {
      "External id": 2481200,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 8903
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571568.251, "dur": 1.417,
    "args": {
      "External id": 2481201,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 8904
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571573.424, "dur": 43.582,
    "args": {
      "External id": 2481202,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 8905
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571574.422, "dur": 42.020,
    "args": {
      "External id": 2481203,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 8906
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571621.083, "dur": 15.284,
    "args": {
      "External id": 2481204,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 8907
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571642.329, "dur": 3.846,
    "args": {
      "External id": 2481205,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 8908
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571644.423, "dur": 0.909,
    "args": {
      "External id": 2481206,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "67108864"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 8909
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571652.004, "dur": 45.767,
    "args": {
      "External id": 2481207,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 8910
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571653.004, "dur": 2.830,
    "args": {
      "External id": 2481208,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 8911
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571653.682, "dur": 1.649,
    "args": {
      "External id": 2481209,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 8912
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571654.704, "dur": 0.475,
    "args": {
      "External id": 2481210,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 8913
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571656.294, "dur": 41.029,
    "args": {
      "External id": 2481211,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 8914
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571656.728, "dur": 40.070,
    "args": {
      "External id": 2481212,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 8915
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571701.511, "dur": 3.772,
    "args": {
      "External id": 2481213,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 8916
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571703.462, "dur": 0.701,
    "args": {
      "External id": 2481214,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "32768"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 8917
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571712.594, "dur": 1.540,
    "args": {
      "External id": 2481215,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 8918
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571720.998, "dur": 5.853,
    "args": {
      "External id": 2481216,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 8919
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571722.574, "dur": 4.029,
    "args": {
      "External id": 2481217,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8920
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571805.126, "dur": 168.639,
    "args": {
      "External id": 2481218,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 8921
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571808.743, "dur": 2.189,
    "args": {
      "External id": 2481219,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8922
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571812.473, "dur": 160.728,
    "args": {
      "External id": 2481220,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 8923
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571815.432, "dur": 0.291,
    "args": {
      "External id": 2481221,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 8924
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571816.597, "dur": 19.679,
    "args": {
      "External id": 2481222,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 8925
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571838.031, "dur": 4.204,
    "args": {
      "External id": 2481223,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 8926
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571841.268, "dur": 0.756,
    "args": {
      "External id": 2481224,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 8927
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571842.991, "dur": 19.953,
    "args": {
      "External id": 2481225,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 8928
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571844.035, "dur": 1.354,
    "args": {
      "External id": 2481226,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8929
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571846.389, "dur": 16.270,
    "args": {
      "External id": 2481227,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 8930
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571848.529, "dur": 2.579,
    "args": {
      "External id": 2481228,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 8931
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571863.989, "dur": 18.946,
    "args": {
      "External id": 2481229,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 8932
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571884.408, "dur": 14.780,
    "args": {
      "External id": 2481230,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 8933
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571903.332, "dur": 12.715,
    "args": {
      "External id": 2481231,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 8934
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571917.298, "dur": 11.319,
    "args": {
      "External id": 2481232,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 8935
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571930.048, "dur": 17.525,
    "args": {
      "External id": 2481233,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 8936
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571931.806, "dur": 1.032,
    "args": {
      "External id": 2481234,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 8937
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571934.484, "dur": 1.028,
    "args": {
      "External id": 2481235,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 8938
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571949.252, "dur": 10.967,
    "args": {
      "External id": 2481236,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 8939
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656571961.179, "dur": 11.078,
    "args": {
      "External id": 2481237,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 8940
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572015.769, "dur": 2.723,
    "args": {
      "External id": 2481238,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 8941
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572029.053, "dur": 3.997,
    "args": {
      "External id": 2481239,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 8942
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572031.472, "dur": 0.742,
    "args": {
      "External id": 2481240,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "32768"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 8943
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572096.994, "dur": 53.488,
    "args": {
      "External id": 2481241,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 8944
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572154.958, "dur": 4.616,
    "args": {
      "External id": 2481242,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 8945
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572157.387, "dur": 1.157,
    "args": {
      "External id": 2481243,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "67108864"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 8946
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572160.895, "dur": 22.343,
    "args": {
      "External id": 2481244,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 8947
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572187.424, "dur": 7.022,
    "args": {
      "External id": 2481245,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 8948
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572190.617, "dur": 3.265,
    "args": {
      "External id": 2481246,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 8949
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572192.515, "dur": 1.149,
    "args": {
      "External id": 2481247,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 8950
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572196.864, "dur": 39.320,
    "args": {
      "External id": 2481248,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 8951
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572197.744, "dur": 37.885,
    "args": {
      "External id": 2481249,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 8952
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572239.986, "dur": 15.136,
    "args": {
      "External id": 2481250,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 8953
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572260.673, "dur": 5.359,
    "args": {
      "External id": 2481251,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 8954
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572264.286, "dur": 0.810,
    "args": {
      "External id": 2481252,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "83886080"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 8955
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572269.495, "dur": 49.939,
    "args": {
      "External id": 2481253,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 8956
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572270.450, "dur": 3.433,
    "args": {
      "External id": 2481254,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 8957
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572271.222, "dur": 2.211,
    "args": {
      "External id": 2481255,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 8958
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572272.580, "dur": 0.717,
    "args": {
      "External id": 2481256,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 8959
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572274.439, "dur": 44.649,
    "args": {
      "External id": 2481257,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 8960
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572275.120, "dur": 43.463,
    "args": {
      "External id": 2481258,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 8961
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572324.773, "dur": 3.780,
    "args": {
      "External id": 2481259,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 8962
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572326.485, "dur": 0.764,
    "args": {
      "External id": 2481260,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "40960"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 8963
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572340.626, "dur": 1.470,
    "args": {
      "External id": 2481261,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 8964
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572353.055, "dur": 7.656,
    "args": {
      "External id": 2481262,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 8965
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572354.792, "dur": 5.613,
    "args": {
      "External id": 2481263,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8966
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572440.523, "dur": 200.517,
    "args": {
      "External id": 2481264,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 8967
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572442.227, "dur": 2.559,
    "args": {
      "External id": 2481265,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8968
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572446.327, "dur": 194.237,
    "args": {
      "External id": 2481266,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 8969
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572455.093, "dur": 0.266,
    "args": {
      "External id": 2481267,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 8970
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572458.228, "dur": 20.210,
    "args": {
      "External id": 2481268,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 8971
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572479.944, "dur": 4.793,
    "args": {
      "External id": 2481269,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 8972
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572482.085, "dur": 2.350,
    "args": {
      "External id": 2481270,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 8973
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572485.485, "dur": 21.304,
    "args": {
      "External id": 2481271,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 8974
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572486.624, "dur": 1.399,
    "args": {
      "External id": 2481272,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8975
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572490.913, "dur": 15.631,
    "args": {
      "External id": 2481273,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 8976
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572493.265, "dur": 2.360,
    "args": {
      "External id": 2481274,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 8977
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572508.293, "dur": 35.811,
    "args": {
      "External id": 2481275,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 8978
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572546.397, "dur": 14.180,
    "args": {
      "External id": 2481276,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 8979
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572563.686, "dur": 14.463,
    "args": {
      "External id": 2481277,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 8980
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572579.624, "dur": 11.947,
    "args": {
      "External id": 2481278,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 8981
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572593.466, "dur": 20.287,
    "args": {
      "External id": 2481279,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 8982
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572595.354, "dur": 1.626,
    "args": {
      "External id": 2481280,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 8983
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572600.493, "dur": 0.853,
    "args": {
      "External id": 2481281,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 8984
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572615.341, "dur": 11.551,
    "args": {
      "External id": 2481282,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 8985
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572627.919, "dur": 11.512,
    "args": {
      "External id": 2481283,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 8986
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572648.497, "dur": 2.136,
    "args": {
      "External id": 2481284,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 8987
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572659.764, "dur": 3.929,
    "args": {
      "External id": 2481285,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 8988
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572662.239, "dur": 0.559,
    "args": {
      "External id": 2481286,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "40960"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 8989
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572728.380, "dur": 59.287,
    "args": {
      "External id": 2481287,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 8990
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572792.248, "dur": 7.441,
    "args": {
      "External id": 2481288,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 8991
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572796.170, "dur": 2.250,
    "args": {
      "External id": 2481289,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "83886080"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 8992
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572801.243, "dur": 24.181,
    "args": {
      "External id": 2481290,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 8993
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572830.455, "dur": 5.438,
    "args": {
      "External id": 2481291,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 8994
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572831.776, "dur": 3.442,
    "args": {
      "External id": 2481292,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 8995
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572833.871, "dur": 1.137,
    "args": {
      "External id": 2481293,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 8996
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572838.515, "dur": 43.890,
    "args": {
      "External id": 2481294,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 8997
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572841.323, "dur": 40.366,
    "args": {
      "External id": 2481295,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 8998
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572886.622, "dur": 14.690,
    "args": {
      "External id": 2481296,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 8999
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572906.434, "dur": 3.618,
    "args": {
      "External id": 2481297,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9000
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572908.331, "dur": 0.887,
    "args": {
      "External id": 2481298,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "100663296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9001
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572913.501, "dur": 49.803,
    "args": {
      "External id": 2481299,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9002
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572914.382, "dur": 5.577,
    "args": {
      "External id": 2481300,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9003
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572915.442, "dur": 3.887,
    "args": {
      "External id": 2481301,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9004
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572918.224, "dur": 0.740,
    "args": {
      "External id": 2481302,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9005
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572920.577, "dur": 42.338,
    "args": {
      "External id": 2481303,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9006
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572921.158, "dur": 41.275,
    "args": {
      "External id": 2481304,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9007
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572967.309, "dur": 3.694,
    "args": {
      "External id": 2481305,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9008
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572969.103, "dur": 0.785,
    "args": {
      "External id": 2481306,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "49152"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9009
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656572976.534, "dur": 1.306,
    "args": {
      "External id": 2481307,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9010
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573025.097, "dur": 10.034,
    "args": {
      "External id": 2481308,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9011
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573028.710, "dur": 6.023,
    "args": {
      "External id": 2481309,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9012
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573119.373, "dur": 173.982,
    "args": {
      "External id": 2481310,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9013
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573121.646, "dur": 3.721,
    "args": {
      "External id": 2481311,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9014
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573126.682, "dur": 166.073,
    "args": {
      "External id": 2481312,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9015
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573128.082, "dur": 0.438,
    "args": {
      "External id": 2481313,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9016
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573130.156, "dur": 20.790,
    "args": {
      "External id": 2481314,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9017
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573152.433, "dur": 5.520,
    "args": {
      "External id": 2481315,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9018
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573156.578, "dur": 1.079,
    "args": {
      "External id": 2481316,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9019
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573160.600, "dur": 19.712,
    "args": {
      "External id": 2481317,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9020
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573161.503, "dur": 1.250,
    "args": {
      "External id": 2481318,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9021
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573163.889, "dur": 16.097,
    "args": {
      "External id": 2481319,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9022
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573166.565, "dur": 2.402,
    "args": {
      "External id": 2481320,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9023
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573181.438, "dur": 20.081,
    "args": {
      "External id": 2481321,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9024
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573202.770, "dur": 12.831,
    "args": {
      "External id": 2481322,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9025
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573218.404, "dur": 12.911,
    "args": {
      "External id": 2481323,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9026
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573232.551, "dur": 11.453,
    "args": {
      "External id": 2481324,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9027
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573245.486, "dur": 20.836,
    "args": {
      "External id": 2481325,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9028
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573248.605, "dur": 1.037,
    "args": {
      "External id": 2481326,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9029
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573251.505, "dur": 2.256,
    "args": {
      "External id": 2481327,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9030
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573267.895, "dur": 11.478,
    "args": {
      "External id": 2481328,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9031
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573280.407, "dur": 11.373,
    "args": {
      "External id": 2481329,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9032
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573299.627, "dur": 2.181,
    "args": {
      "External id": 2481330,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9033
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573309.484, "dur": 3.724,
    "args": {
      "External id": 2481331,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9034
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573311.966, "dur": 0.440,
    "args": {
      "External id": 2481332,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "49152"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9035
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573372.809, "dur": 55.157,
    "args": {
      "External id": 2481333,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9036
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573434.651, "dur": 4.331,
    "args": {
      "External id": 2481334,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9037
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573437.213, "dur": 0.694,
    "args": {
      "External id": 2481335,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "100663296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9038
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573440.397, "dur": 23.197,
    "args": {
      "External id": 2481336,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9039
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573468.053, "dur": 5.516,
    "args": {
      "External id": 2481337,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9040
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573469.645, "dur": 3.252,
    "args": {
      "External id": 2481338,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9041
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573471.389, "dur": 1.331,
    "args": {
      "External id": 2481339,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9042
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573477.601, "dur": 40.478,
    "args": {
      "External id": 2481340,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9043
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573478.470, "dur": 39.019,
    "args": {
      "External id": 2481341,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9044
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573521.495, "dur": 30.415,
    "args": {
      "External id": 2481342,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9045
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573559.606, "dur": 4.842,
    "args": {
      "External id": 2481343,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9046
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573562.244, "dur": 1.127,
    "args": {
      "External id": 2481344,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "117440512"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9047
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573568.498, "dur": 52.817,
    "args": {
      "External id": 2481345,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9048
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573569.209, "dur": 5.636,
    "args": {
      "External id": 2481346,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9049
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573572.251, "dur": 2.075,
    "args": {
      "External id": 2481347,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9050
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573573.579, "dur": 0.595,
    "args": {
      "External id": 2481348,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9051
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573575.598, "dur": 45.320,
    "args": {
      "External id": 2481349,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9052
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573576.369, "dur": 43.840,
    "args": {
      "External id": 2481350,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9053
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573625.042, "dur": 3.848,
    "args": {
      "External id": 2481351,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9054
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573626.931, "dur": 0.792,
    "args": {
      "External id": 2481352,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "57344"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9055
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573636.379, "dur": 1.288,
    "args": {
      "External id": 2481353,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9056
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573644.322, "dur": 5.875,
    "args": {
      "External id": 2481354,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9057
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573646.062, "dur": 3.868,
    "args": {
      "External id": 2481355,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9058
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573724.633, "dur": 173.731,
    "args": {
      "External id": 2481356,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9059
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573728.541, "dur": 2.091,
    "args": {
      "External id": 2481357,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9060
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573732.080, "dur": 165.657,
    "args": {
      "External id": 2481358,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9061
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573735.194, "dur": 0.439,
    "args": {
      "External id": 2481359,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9062
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573736.571, "dur": 20.059,
    "args": {
      "External id": 2481360,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9063
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573758.264, "dur": 4.718,
    "args": {
      "External id": 2481361,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9064
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573761.548, "dur": 1.208,
    "args": {
      "External id": 2481362,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9065
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573763.865, "dur": 19.663,
    "args": {
      "External id": 2481363,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9066
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573764.940, "dur": 1.236,
    "args": {
      "External id": 2481364,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9067
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573767.388, "dur": 15.907,
    "args": {
      "External id": 2481365,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9068
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573769.756, "dur": 2.453,
    "args": {
      "External id": 2481366,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9069
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573784.714, "dur": 18.893,
    "args": {
      "External id": 2481367,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9070
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573805.230, "dur": 16.572,
    "args": {
      "External id": 2481368,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9071
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573825.859, "dur": 14.336,
    "args": {
      "External id": 2481369,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9072
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573841.529, "dur": 11.466,
    "args": {
      "External id": 2481370,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9073
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573854.562, "dur": 17.309,
    "args": {
      "External id": 2481371,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9074
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573856.571, "dur": 1.198,
    "args": {
      "External id": 2481372,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9075
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573859.612, "dur": 0.726,
    "args": {
      "External id": 2481373,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9076
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573873.627, "dur": 11.213,
    "args": {
      "External id": 2481374,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9077
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573885.936, "dur": 11.008,
    "args": {
      "External id": 2481375,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9078
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573905.449, "dur": 1.460,
    "args": {
      "External id": 2481376,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9079
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573914.335, "dur": 3.011,
    "args": {
      "External id": 2481377,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9080
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573916.062, "dur": 0.518,
    "args": {
      "External id": 2481378,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "57344"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9081
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656573972.521, "dur": 86.764,
    "args": {
      "External id": 2481379,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9082
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574066.422, "dur": 5.382,
    "args": {
      "External id": 2481380,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9083
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574069.409, "dur": 0.899,
    "args": {
      "External id": 2481381,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "117440512"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9084
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574073.283, "dur": 25.523,
    "args": {
      "External id": 2481382,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9085
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574103.826, "dur": 7.120,
    "args": {
      "External id": 2481383,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9086
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574107.103, "dur": 3.109,
    "args": {
      "External id": 2481384,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9087
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574109.034, "dur": 0.965,
    "args": {
      "External id": 2481385,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9088
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574114.154, "dur": 41.902,
    "args": {
      "External id": 2481386,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9089
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574115.199, "dur": 40.162,
    "args": {
      "External id": 2481387,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9090
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574159.696, "dur": 14.083,
    "args": {
      "External id": 2481388,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9091
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574178.536, "dur": 25.659,
    "args": {
      "External id": 2481389,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", ""], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 9092
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574180.911, "dur": 22.904,
    "args": {
      "External id": 2481390,"Record function id": 0, "Concrete Inputs": ["", "[]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9093
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574186.175, "dur": 2.260,
    "args": {
      "External id": 2481391,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 9094
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574212.044, "dur": 26.838,
    "args": {
      "External id": 2481392,"Record function id": 0, "Concrete Inputs": ["", "", "15", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], []], "Ev Idx": 9095
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574213.922, "dur": 24.738,
    "args": {
      "External id": 2481393,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], [], []], "Ev Idx": 9096
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574218.517, "dur": 4.023,
    "args": {
      "External id": 2481394,"Record function id": 0, "Concrete Inputs": ["[32000, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9097
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574223.781, "dur": 14.453,
    "args": {
      "External id": 2481395,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "float", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9098
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574252.345, "dur": 5.044,
    "args": {
      "External id": 2481396,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 9099
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574254.085, "dur": 3.000,
    "args": {
      "External id": 2481397,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 9100
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574258.321, "dur": 1.142,
    "args": {
      "External id": 2481398,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9101
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574258.918, "dur": 0.476,
    "args": {
      "External id": 2481399,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9102
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574297.549, "dur": 20.992,
    "args": {
      "External id": 2481400,"Sequence number": 24740625, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[1], [], []], "Ev Idx": 9103
    }
  },
  {
    "ph": "s", "id": 23, "pid": 1336754, "tid": 1336754, "ts": 1514656574297.549,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574324.250, "dur": 6.085,
    "args": {
      "External id": 2481401,"Sequence number": 24740626, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], [], []], "Input Dims": [[16, 4, 4096], [], [], [], []], "Ev Idx": 9104
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574328.267, "dur": 0.941,
    "args": {
      "External id": 2481402,"Record function id": 0, "Concrete Inputs": ["", "[16, 4, 4096]", "[8192, 1, 1]", "1"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 4, 4096], [], [], []], "Ev Idx": 9105
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574332.680, "dur": 6.103,
    "args": {
      "External id": 2481403,"Sequence number": 24740626, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "1"], "Input type": ["long int", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], []], "Input Dims": [[16, 4, 4096], [], []], "Ev Idx": 9106
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574336.576, "dur": 0.968,
    "args": {
      "External id": 2481404,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096]", "[8192, 1]", "2"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 4, 4096], [], [], []], "Ev Idx": 9107
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574339.946, "dur": 4.688,
    "args": {
      "External id": 2481405,"Sequence number": 24740626, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "0", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], []], "Ev Idx": 9108
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574343.356, "dur": 0.716,
    "args": {
      "External id": 2481406,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096]", "[8192, 1]", "2"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1], [], [], []], "Input Dims": [[16, 4096], [], [], []], "Ev Idx": 9109
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574348.730, "dur": 5.806,
    "args": {
      "External id": 2481407,"Sequence number": 24740626, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 9110
    }
  },
  {
    "ph": "s", "id": 22, "pid": 1336754, "tid": 1336754, "ts": 1514656574348.730,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574352.194, "dur": 1.037,
    "args": {
      "External id": 2481408,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 9111
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574355.534, "dur": 4.321,
    "args": {
      "External id": 2481409,"Sequence number": 24740627, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 9112
    }
  },
  {
    "ph": "s", "id": 21, "pid": 1336754, "tid": 1336754, "ts": 1514656574355.534,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574358.418, "dur": 0.647,
    "args": {
      "External id": 2481410,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 9113
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574362.514, "dur": 4.639,
    "args": {
      "External id": 2481411,"Sequence number": 24740628, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "2", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], []], "Input Dims": [[16, 4096, 4, 2048], [], []], "Ev Idx": 9114
    }
  },
  {
    "ph": "s", "id": 20, "pid": 1336754, "tid": 1336754, "ts": 1514656574362.514,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574365.636, "dur": 0.630,
    "args": {
      "External id": 2481412,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "2048"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 9115
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574368.320, "dur": 4.870,
    "args": {
      "External id": 2481413,"Sequence number": 24740629, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "2", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 1], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], []], "Ev Idx": 9116
    }
  },
  {
    "ph": "s", "id": 19, "pid": 1336754, "tid": 1336754, "ts": 1514656574368.320,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574371.390, "dur": 0.999,
    "args": {
      "External id": 2481414,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "2048"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 1], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], []], "Ev Idx": 9117
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::contiguous", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574377.224, "dur": 29.847,
    "args": {
      "External id": 2481415,"Sequence number": 24740630, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["long int", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 9118
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574379.188, "dur": 27.587,
    "args": {
      "External id": 2481416,"Sequence number": 24740630, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["long int", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 9119
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574381.705, "dur": 7.956,
    "args": {
      "External id": 2481417,"Record function id": 0, "Concrete Inputs": ["", "4", "0", "", "", "0"], "Input type": ["long int", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[8192, 1], [], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], [], []], "Ev Idx": 9120
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574385.815, "dur": 3.204,
    "args": {
      "External id": 2481418,"Record function id": 0, "Concrete Inputs": ["[16, 4096]", "4", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9121
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574390.461, "dur": 15.844,
    "args": {
      "External id": 2481419,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[4096, 1], [8192, 1], []], "Input Dims": [[16, 4096], [16, 4096], []], "Ev Idx": 9122
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574433.461, "dur": 4.006,
    "args": {
      "External id": 2481420,"Sequence number": 24740630, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 9123
    }
  },
  {
    "ph": "s", "id": 18, "pid": 1336754, "tid": 1336754, "ts": 1514656574433.461,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574439.763, "dur": 1.360,
    "args": {
      "External id": 2481421,"Sequence number": 24740631, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["long int", "ScalarList"], "Input Strides": [[4096, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 9124
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FusedLinearCrossEntropyFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574471.430, "dur": 42694.287,
    "args": {
      "External id": 2481422,"Sequence number": 24740631, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "-100", "0.", "1.", "8"], "Input type": ["c10::BFloat16", "long int", "c10::BFloat16", "", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1], [1], [2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536], [32000, 2048], [], [], [], [], []], "Ev Idx": 9125
    }
  },
  {
    "ph": "s", "id": 17, "pid": 1336754, "tid": 1336754, "ts": 1514656574471.430,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::contiguous", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574485.962, "dur": 25.654,
    "args": {
      "External id": 2481423,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 9126
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574486.605, "dur": 24.811,
    "args": {
      "External id": 2481424,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 9127
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574487.752, "dur": 6.982,
    "args": {
      "External id": 2481425,"Record function id": 0, "Concrete Inputs": ["", "15", "0", "", "", "0"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[8192, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 9128
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574490.903, "dur": 3.326,
    "args": {
      "External id": 2481426,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9129
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574495.474, "dur": 15.555,
    "args": {
      "External id": 2481427,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [8192, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 9130
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574526.645, "dur": 47.314,
    "args": {
      "External id": 2481428,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 9131
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574545.366, "dur": 7.165,
    "args": {
      "External id": 2481429,"Record function id": 0, "Concrete Inputs": ["", "15", "0", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 9132
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574547.320, "dur": 4.751,
    "args": {
      "External id": 2481430,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9133
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574553.766, "dur": 19.974,
    "args": {
      "External id": 2481431,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 9134
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574555.653, "dur": 17.654,
    "args": {
      "External id": 2481432,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 9135
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574578.319, "dur": 18.474,
    "args": {
      "External id": 2481433,"Record function id": 0, "Concrete Inputs": ["", "6", "", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], []], "Ev Idx": 9136
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574578.945, "dur": 4.922,
    "args": {
      "External id": 2481434,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], []], "Ev Idx": 9137
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574580.609, "dur": 2.995,
    "args": {
      "External id": 2481435,"Record function id": 0, "Concrete Inputs": ["[32000, 2048]", "[2048, 1]", "6", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9138
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574584.563, "dur": 12.052,
    "args": {
      "External id": 2481436,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9139
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574585.302, "dur": 10.810,
    "args": {
      "External id": 2481437,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[2048, 1], []], "Input Dims": [[32000, 2048], []], "Ev Idx": 9140
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574603.924, "dur": 16.803,
    "args": {
      "External id": 2481438,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False"], "Input type": ["ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 9141
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574605.771, "dur": 3.491,
    "args": {
      "External id": 2481439,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9142
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574610.231, "dur": 10.244,
    "args": {
      "External id": 2481440,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[65536]], "Ev Idx": 9143
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574611.018, "dur": 9.176,
    "args": {
      "External id": 2481441,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 9144
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::ne", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574625.162, "dur": 20.941,
    "args": {
      "External id": 2481442,"Record function id": 0, "Concrete Inputs": ["", "-100"], "Input type": ["long int", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 9145
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574648.793, "dur": 64.205,
    "args": {
      "External id": 2481443,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["bool", ""], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 9146
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574666.706, "dur": 45.866,
    "args": {
      "External id": 2481444,"Record function id": 0, "Concrete Inputs": ["", "[]", "False", ""], "Input type": ["bool", "ScalarList", "Scalar", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9147
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574671.884, "dur": 2.150,
    "args": {
      "External id": 2481445,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[0]", ""], "Input type": ["long int", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 9148
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574675.171, "dur": 22.802,
    "args": {
      "External id": 2481446,"Record function id": 0, "Concrete Inputs": ["", "4", "False", "False", ""], "Input type": ["bool", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9149
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574676.744, "dur": 20.951,
    "args": {
      "External id": 2481447,"Record function id": 0, "Concrete Inputs": ["", "4", "", "", "", "False", ""], "Input type": ["bool", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[65536], [], [], [], [], [], []], "Ev Idx": 9150
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574678.895, "dur": 2.506,
    "args": {
      "External id": 2481448,"Record function id": 0, "Concrete Inputs": ["[65536]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9151
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574683.691, "dur": 13.666,
    "args": {
      "External id": 2481449,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["long int", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[65536], [65536], []], "Ev Idx": 9152
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::item", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574716.694, "dur": 36932.290,
    "args": {
      "External id": 2481450,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["long int"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 9153
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_local_scalar_dense", "pid": 1336754, "tid": 1336754,
    "ts": 1514656574718.410, "dur": 36929.294,
    "args": {
      "External id": 2481451,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["long int"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 9154
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656611660.770, "dur": 6.635,
    "args": {
      "External id": 2481452,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9155
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656611664.501, "dur": 1.260,
    "args": {
      "External id": 2481453,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9156
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656611672.575, "dur": 104.689,
    "args": {
      "External id": 2481454,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9157
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656611673.956, "dur": 8.400,
    "args": {
      "External id": 2481455,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9158
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656611678.340, "dur": 3.260,
    "args": {
      "External id": 2481456,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9159
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656611680.242, "dur": 1.092,
    "args": {
      "External id": 2481457,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9160
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656611683.709, "dur": 92.829,
    "args": {
      "External id": 2481458,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9161
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656611685.196, "dur": 90.444,
    "args": {
      "External id": 2481459,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9162
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656611780.935, "dur": 4.151,
    "args": {
      "External id": 2481460,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9163
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656611782.864, "dur": 1.037,
    "args": {
      "External id": 2481461,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "0"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9164
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656611792.123, "dur": 2.233,
    "args": {
      "External id": 2481462,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9165
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656611804.727, "dur": 7.009,
    "args": {
      "External id": 2481463,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9166
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656611806.798, "dur": 4.642,
    "args": {
      "External id": 2481464,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9167
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656611938.111, "dur": 225.933,
    "args": {
      "External id": 2481465,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9168
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656611942.736, "dur": 2.105,
    "args": {
      "External id": 2481466,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9169
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656611946.209, "dur": 217.254,
    "args": {
      "External id": 2481467,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9170
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336754, "tid": 1336754,
    "ts": 1514656611947.774, "dur": 0.544,
    "args": {
      "External id": 2481468,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9171
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336754, "tid": 1336754,
    "ts": 1514656611951.966, "dur": 24.653,
    "args": {
      "External id": 2481469,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9172
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514656611978.415, "dur": 32.661,
    "args": {
      "External id": 2481470,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9173
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612009.110, "dur": 1.155,
    "args": {
      "External id": 2481471,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9174
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612012.199, "dur": 26.541,
    "args": {
      "External id": 2481472,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9175
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612013.538, "dur": 2.414,
    "args": {
      "External id": 2481473,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9176
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612017.133, "dur": 21.256,
    "args": {
      "External id": 2481474,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9177
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612021.330, "dur": 3.309,
    "args": {
      "External id": 2481475,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9178
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612040.318, "dur": 21.465,
    "args": {
      "External id": 2481476,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9179
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612063.431, "dur": 16.893,
    "args": {
      "External id": 2481477,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9180
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612083.349, "dur": 14.298,
    "args": {
      "External id": 2481478,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9181
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612101.104, "dur": 12.831,
    "args": {
      "External id": 2481479,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9182
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612115.975, "dur": 19.567,
    "args": {
      "External id": 2481480,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9183
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612117.799, "dur": 1.545,
    "args": {
      "External id": 2481481,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9184
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612121.860, "dur": 0.699,
    "args": {
      "External id": 2481482,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9185
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612137.720, "dur": 12.529,
    "args": {
      "External id": 2481483,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9186
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612151.573, "dur": 10.679,
    "args": {
      "External id": 2481484,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9187
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612174.291, "dur": 2.391,
    "args": {
      "External id": 2481485,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9188
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612184.333, "dur": 4.572,
    "args": {
      "External id": 2481486,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9189
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612186.960, "dur": 0.811,
    "args": {
      "External id": 2481487,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9190
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612265.970, "dur": 68.200,
    "args": {
      "External id": 2481488,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9191
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612339.618, "dur": 5.985,
    "args": {
      "External id": 2481489,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9192
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612342.040, "dur": 1.030,
    "args": {
      "External id": 2481490,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9193
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612347.095, "dur": 32.147,
    "args": {
      "External id": 2481491,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9194
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612385.172, "dur": 8.414,
    "args": {
      "External id": 2481492,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9195
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612388.725, "dur": 4.133,
    "args": {
      "External id": 2481493,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9196
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612390.984, "dur": 1.648,
    "args": {
      "External id": 2481494,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9197
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612397.256, "dur": 44.683,
    "args": {
      "External id": 2481495,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9198
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612398.316, "dur": 43.099,
    "args": {
      "External id": 2481496,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9199
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612446.704, "dur": 14.525,
    "args": {
      "External id": 2481497,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9200
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612467.921, "dur": 4.561,
    "args": {
      "External id": 2481498,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9201
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612471.103, "dur": 0.497,
    "args": {
      "External id": 2481499,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "16777216"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9202
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612476.935, "dur": 68.265,
    "args": {
      "External id": 2481500,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9203
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612477.934, "dur": 4.043,
    "args": {
      "External id": 2481501,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9204
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612478.898, "dur": 2.560,
    "args": {
      "External id": 2481502,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9205
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612480.554, "dur": 0.749,
    "args": {
      "External id": 2481503,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9206
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612482.560, "dur": 62.126,
    "args": {
      "External id": 2481504,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9207
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612487.665, "dur": 56.104,
    "args": {
      "External id": 2481505,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9208
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612552.721, "dur": 4.782,
    "args": {
      "External id": 2481506,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9209
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612554.971, "dur": 1.293,
    "args": {
      "External id": 2481507,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "8192"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9210
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612563.967, "dur": 1.599,
    "args": {
      "External id": 2481508,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9211
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612573.557, "dur": 8.536,
    "args": {
      "External id": 2481509,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9212
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612575.668, "dur": 6.127,
    "args": {
      "External id": 2481510,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9213
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612676.472, "dur": 184.279,
    "args": {
      "External id": 2481511,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9214
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612678.393, "dur": 2.400,
    "args": {
      "External id": 2481512,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9215
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612682.233, "dur": 177.965,
    "args": {
      "External id": 2481513,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9216
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612683.780, "dur": 0.412,
    "args": {
      "External id": 2481514,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9217
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612685.235, "dur": 23.859,
    "args": {
      "External id": 2481515,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9218
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612712.665, "dur": 6.858,
    "args": {
      "External id": 2481516,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9219
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612718.291, "dur": 0.917,
    "args": {
      "External id": 2481517,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9220
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612720.424, "dur": 24.691,
    "args": {
      "External id": 2481518,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9221
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612723.286, "dur": 2.925,
    "args": {
      "External id": 2481519,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9222
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612727.360, "dur": 17.364,
    "args": {
      "External id": 2481520,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9223
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612730.344, "dur": 2.827,
    "args": {
      "External id": 2481521,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9224
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612746.623, "dur": 20.546,
    "args": {
      "External id": 2481522,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9225
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612768.629, "dur": 12.944,
    "args": {
      "External id": 2481523,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9226
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612784.682, "dur": 13.833,
    "args": {
      "External id": 2481524,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9227
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612800.270, "dur": 12.001,
    "args": {
      "External id": 2481525,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9228
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612814.123, "dur": 20.643,
    "args": {
      "External id": 2481526,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9229
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612816.172, "dur": 1.313,
    "args": {
      "External id": 2481527,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9230
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612821.380, "dur": 0.699,
    "args": {
      "External id": 2481528,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9231
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612836.331, "dur": 11.274,
    "args": {
      "External id": 2481529,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9232
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612848.851, "dur": 10.567,
    "args": {
      "External id": 2481530,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9233
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612867.222, "dur": 1.848,
    "args": {
      "External id": 2481531,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9234
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612877.603, "dur": 3.606,
    "args": {
      "External id": 2481532,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9235
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612880.018, "dur": 0.424,
    "args": {
      "External id": 2481533,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "8192"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9236
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656612944.834, "dur": 89.834,
    "args": {
      "External id": 2481534,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9237
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613041.817, "dur": 7.598,
    "args": {
      "External id": 2481535,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9238
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613046.483, "dur": 1.448,
    "args": {
      "External id": 2481536,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "16777216"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9239
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613050.833, "dur": 26.077,
    "args": {
      "External id": 2481537,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9240
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613082.505, "dur": 6.835,
    "args": {
      "External id": 2481538,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9241
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613083.960, "dur": 4.632,
    "args": {
      "External id": 2481539,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9242
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613085.855, "dur": 2.492,
    "args": {
      "External id": 2481540,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9243
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613092.346, "dur": 47.871,
    "args": {
      "External id": 2481541,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9244
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613095.129, "dur": 44.542,
    "args": {
      "External id": 2481542,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9245
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613144.362, "dur": 14.858,
    "args": {
      "External id": 2481543,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9246
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613165.635, "dur": 3.299,
    "args": {
      "External id": 2481544,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9247
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613167.397, "dur": 0.676,
    "args": {
      "External id": 2481545,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "33554432"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9248
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613172.736, "dur": 49.093,
    "args": {
      "External id": 2481546,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9249
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613173.429, "dur": 5.120,
    "args": {
      "External id": 2481547,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9250
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613174.271, "dur": 3.693,
    "args": {
      "External id": 2481548,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9251
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613176.975, "dur": 0.814,
    "args": {
      "External id": 2481549,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9252
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613179.127, "dur": 42.299,
    "args": {
      "External id": 2481550,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9253
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613179.880, "dur": 40.993,
    "args": {
      "External id": 2481551,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9254
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613226.027, "dur": 3.198,
    "args": {
      "External id": 2481552,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9255
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613227.652, "dur": 0.588,
    "args": {
      "External id": 2481553,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "16384"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9256
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613235.128, "dur": 1.403,
    "args": {
      "External id": 2481554,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9257
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613244.059, "dur": 8.536,
    "args": {
      "External id": 2481555,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9258
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613247.662, "dur": 4.688,
    "args": {
      "External id": 2481556,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9259
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613337.245, "dur": 171.919,
    "args": {
      "External id": 2481557,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9260
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613339.518, "dur": 2.346,
    "args": {
      "External id": 2481558,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9261
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613344.875, "dur": 163.803,
    "args": {
      "External id": 2481559,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9262
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613346.069, "dur": 0.405,
    "args": {
      "External id": 2481560,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9263
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613347.566, "dur": 19.354,
    "args": {
      "External id": 2481561,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9264
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613368.277, "dur": 4.676,
    "args": {
      "External id": 2481562,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9265
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613370.269, "dur": 2.408,
    "args": {
      "External id": 2481563,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9266
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613373.777, "dur": 23.640,
    "args": {
      "External id": 2481564,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9267
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613376.357, "dur": 1.365,
    "args": {
      "External id": 2481565,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9268
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613378.857, "dur": 18.294,
    "args": {
      "External id": 2481566,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9269
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613381.352, "dur": 2.344,
    "args": {
      "External id": 2481567,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9270
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613398.725, "dur": 18.646,
    "args": {
      "External id": 2481568,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9271
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613418.767, "dur": 12.741,
    "args": {
      "External id": 2481569,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9272
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613434.123, "dur": 13.033,
    "args": {
      "External id": 2481570,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9273
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613448.748, "dur": 11.680,
    "args": {
      "External id": 2481571,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9274
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613462.081, "dur": 21.036,
    "args": {
      "External id": 2481572,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9275
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613463.401, "dur": 1.599,
    "args": {
      "External id": 2481573,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9276
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613468.680, "dur": 2.194,
    "args": {
      "External id": 2481574,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9277
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613484.478, "dur": 11.187,
    "args": {
      "External id": 2481575,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9278
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613496.962, "dur": 10.881,
    "args": {
      "External id": 2481576,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9279
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613515.456, "dur": 1.502,
    "args": {
      "External id": 2481577,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9280
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613525.237, "dur": 21.385,
    "args": {
      "External id": 2481578,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9281
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613543.981, "dur": 0.982,
    "args": {
      "External id": 2481579,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "16384"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9282
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613612.561, "dur": 73.273,
    "args": {
      "External id": 2481580,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9283
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613692.386, "dur": 4.730,
    "args": {
      "External id": 2481581,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9284
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613695.050, "dur": 0.746,
    "args": {
      "External id": 2481582,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "33554432"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9285
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613698.513, "dur": 30.988,
    "args": {
      "External id": 2481583,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9286
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613734.014, "dur": 5.617,
    "args": {
      "External id": 2481584,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9287
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613735.405, "dur": 3.507,
    "args": {
      "External id": 2481585,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9288
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613737.101, "dur": 1.596,
    "args": {
      "External id": 2481586,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9289
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613743.709, "dur": 73.175,
    "args": {
      "External id": 2481587,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9290
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613744.814, "dur": 71.497,
    "args": {
      "External id": 2481588,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9291
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613820.797, "dur": 34.509,
    "args": {
      "External id": 2481589,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9292
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613861.075, "dur": 3.580,
    "args": {
      "External id": 2481590,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9293
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613862.987, "dur": 0.757,
    "args": {
      "External id": 2481591,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "50331648"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9294
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613868.193, "dur": 51.116,
    "args": {
      "External id": 2481592,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9295
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613868.922, "dur": 5.206,
    "args": {
      "External id": 2481593,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9296
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613871.273, "dur": 2.395,
    "args": {
      "External id": 2481594,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9297
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613872.905, "dur": 0.639,
    "args": {
      "External id": 2481595,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9298
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613874.735, "dur": 44.233,
    "args": {
      "External id": 2481596,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9299
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613875.353, "dur": 42.955,
    "args": {
      "External id": 2481597,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9300
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613922.790, "dur": 3.283,
    "args": {
      "External id": 2481598,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9301
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613924.467, "dur": 0.661,
    "args": {
      "External id": 2481599,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "24576"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9302
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613933.432, "dur": 1.259,
    "args": {
      "External id": 2481600,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9303
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613941.541, "dur": 5.983,
    "args": {
      "External id": 2481601,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9304
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656613943.122, "dur": 4.139,
    "args": {
      "External id": 2481602,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9305
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614064.734, "dur": 174.277,
    "args": {
      "External id": 2481603,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9306
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614068.854, "dur": 3.478,
    "args": {
      "External id": 2481604,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9307
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614073.978, "dur": 164.612,
    "args": {
      "External id": 2481605,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9308
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614076.887, "dur": 0.416,
    "args": {
      "External id": 2481606,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9309
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614078.382, "dur": 21.170,
    "args": {
      "External id": 2481607,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9310
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614100.914, "dur": 5.633,
    "args": {
      "External id": 2481608,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9311
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614105.210, "dur": 0.957,
    "args": {
      "External id": 2481609,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9312
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614107.386, "dur": 20.003,
    "args": {
      "External id": 2481610,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9313
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614108.393, "dur": 1.491,
    "args": {
      "External id": 2481611,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9314
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614111.071, "dur": 16.032,
    "args": {
      "External id": 2481612,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9315
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614113.565, "dur": 2.522,
    "args": {
      "External id": 2481613,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9316
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614128.587, "dur": 19.210,
    "args": {
      "External id": 2481614,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9317
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614149.147, "dur": 14.467,
    "args": {
      "External id": 2481615,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9318
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614167.754, "dur": 13.386,
    "args": {
      "External id": 2481616,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9319
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614182.426, "dur": 11.571,
    "args": {
      "External id": 2481617,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9320
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614195.870, "dur": 17.021,
    "args": {
      "External id": 2481618,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9321
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614197.306, "dur": 1.185,
    "args": {
      "External id": 2481619,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9322
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614200.085, "dur": 0.572,
    "args": {
      "External id": 2481620,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9323
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614214.299, "dur": 11.225,
    "args": {
      "External id": 2481621,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9324
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614226.702, "dur": 11.007,
    "args": {
      "External id": 2481622,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9325
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614247.269, "dur": 1.794,
    "args": {
      "External id": 2481623,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9326
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614258.084, "dur": 3.609,
    "args": {
      "External id": 2481624,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9327
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614260.299, "dur": 0.456,
    "args": {
      "External id": 2481625,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "24576"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9328
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614325.304, "dur": 57.559,
    "args": {
      "External id": 2481626,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9329
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614387.424, "dur": 4.664,
    "args": {
      "External id": 2481627,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9330
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614389.859, "dur": 1.176,
    "args": {
      "External id": 2481628,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "50331648"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9331
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614393.283, "dur": 23.318,
    "args": {
      "External id": 2481629,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9332
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614421.215, "dur": 7.727,
    "args": {
      "External id": 2481630,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9333
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614424.396, "dur": 3.952,
    "args": {
      "External id": 2481631,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9334
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614426.121, "dur": 2.044,
    "args": {
      "External id": 2481632,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9335
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614431.536, "dur": 41.009,
    "args": {
      "External id": 2481633,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9336
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614432.698, "dur": 39.205,
    "args": {
      "External id": 2481634,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9337
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614476.369, "dur": 13.962,
    "args": {
      "External id": 2481635,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9338
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614496.230, "dur": 5.201,
    "args": {
      "External id": 2481636,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9339
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614499.529, "dur": 1.189,
    "args": {
      "External id": 2481637,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "67108864"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9340
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614505.046, "dur": 66.376,
    "args": {
      "External id": 2481638,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9341
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614505.824, "dur": 3.400,
    "args": {
      "External id": 2481639,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9342
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614506.568, "dur": 2.123,
    "args": {
      "External id": 2481640,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9343
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614507.912, "dur": 0.644,
    "args": {
      "External id": 2481641,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9344
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614509.875, "dur": 61.066,
    "args": {
      "External id": 2481642,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9345
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614510.497, "dur": 59.230,
    "args": {
      "External id": 2481643,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9346
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614577.684, "dur": 4.509,
    "args": {
      "External id": 2481644,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9347
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614579.943, "dur": 0.974,
    "args": {
      "External id": 2481645,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "32768"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9348
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614590.350, "dur": 1.609,
    "args": {
      "External id": 2481646,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9349
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614599.489, "dur": 7.960,
    "args": {
      "External id": 2481647,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9350
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614601.199, "dur": 5.986,
    "args": {
      "External id": 2481648,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9351
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614691.326, "dur": 201.650,
    "args": {
      "External id": 2481649,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9352
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614693.572, "dur": 2.250,
    "args": {
      "External id": 2481650,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9353
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614698.716, "dur": 193.725,
    "args": {
      "External id": 2481651,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9354
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614700.141, "dur": 0.386,
    "args": {
      "External id": 2481652,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9355
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614701.589, "dur": 22.897,
    "args": {
      "External id": 2481653,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9356
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614725.705, "dur": 3.055,
    "args": {
      "External id": 2481654,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9357
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614727.717, "dur": 0.768,
    "args": {
      "External id": 2481655,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9358
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614729.625, "dur": 21.242,
    "args": {
      "External id": 2481656,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9359
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614730.546, "dur": 1.462,
    "args": {
      "External id": 2481657,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9360
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614733.177, "dur": 17.366,
    "args": {
      "External id": 2481658,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9361
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614736.924, "dur": 2.481,
    "args": {
      "External id": 2481659,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9362
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614754.026, "dur": 18.779,
    "args": {
      "External id": 2481660,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9363
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614774.109, "dur": 12.385,
    "args": {
      "External id": 2481661,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9364
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614788.994, "dur": 12.840,
    "args": {
      "External id": 2481662,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9365
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614802.932, "dur": 11.178,
    "args": {
      "External id": 2481663,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9366
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614815.740, "dur": 50.175,
    "args": {
      "External id": 2481664,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9367
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614849.431, "dur": 1.691,
    "args": {
      "External id": 2481665,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9368
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614853.176, "dur": 0.739,
    "args": {
      "External id": 2481666,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9369
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614869.146, "dur": 10.588,
    "args": {
      "External id": 2481667,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9370
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614880.811, "dur": 10.727,
    "args": {
      "External id": 2481668,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9371
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614899.218, "dur": 1.546,
    "args": {
      "External id": 2481669,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9372
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614908.890, "dur": 3.509,
    "args": {
      "External id": 2481670,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9373
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614911.197, "dur": 0.400,
    "args": {
      "External id": 2481671,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "32768"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9374
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656614974.798, "dur": 107.250,
    "args": {
      "External id": 2481672,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9375
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615089.463, "dur": 7.844,
    "args": {
      "External id": 2481673,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9376
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615092.668, "dur": 3.067,
    "args": {
      "External id": 2481674,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "67108864"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9377
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615100.529, "dur": 23.715,
    "args": {
      "External id": 2481675,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9378
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615129.849, "dur": 5.544,
    "args": {
      "External id": 2481676,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9379
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615131.327, "dur": 3.245,
    "args": {
      "External id": 2481677,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9380
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615133.348, "dur": 0.915,
    "args": {
      "External id": 2481678,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9381
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615138.279, "dur": 41.270,
    "args": {
      "External id": 2481679,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9382
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615139.200, "dur": 39.692,
    "args": {
      "External id": 2481680,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9383
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615183.680, "dur": 14.716,
    "args": {
      "External id": 2481681,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9384
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615206.354, "dur": 3.375,
    "args": {
      "External id": 2481682,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9385
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615208.213, "dur": 0.610,
    "args": {
      "External id": 2481683,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "83886080"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9386
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615213.792, "dur": 47.790,
    "args": {
      "External id": 2481684,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9387
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615214.616, "dur": 3.704,
    "args": {
      "External id": 2481685,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9388
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615215.352, "dur": 2.452,
    "args": {
      "External id": 2481686,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9389
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615216.696, "dur": 0.979,
    "args": {
      "External id": 2481687,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9390
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615220.555, "dur": 40.667,
    "args": {
      "External id": 2481688,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9391
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615221.020, "dur": 39.680,
    "args": {
      "External id": 2481689,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9392
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615265.034, "dur": 3.474,
    "args": {
      "External id": 2481690,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9393
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615266.723, "dur": 0.697,
    "args": {
      "External id": 2481691,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "40960"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9394
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615274.316, "dur": 1.406,
    "args": {
      "External id": 2481692,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9395
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615282.942, "dur": 8.234,
    "args": {
      "External id": 2481693,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9396
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615286.632, "dur": 4.301,
    "args": {
      "External id": 2481694,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9397
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615374.983, "dur": 189.441,
    "args": {
      "External id": 2481695,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9398
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615377.032, "dur": 3.626,
    "args": {
      "External id": 2481696,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9399
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615382.066, "dur": 181.879,
    "args": {
      "External id": 2481697,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9400
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615383.136, "dur": 0.257,
    "args": {
      "External id": 2481698,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9401
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615384.586, "dur": 22.821,
    "args": {
      "External id": 2481699,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9402
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615408.995, "dur": 4.757,
    "args": {
      "External id": 2481700,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9403
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615412.798, "dur": 0.680,
    "args": {
      "External id": 2481701,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9404
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615414.659, "dur": 20.577,
    "args": {
      "External id": 2481702,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9405
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615417.138, "dur": 1.592,
    "args": {
      "External id": 2481703,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9406
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615419.724, "dur": 15.124,
    "args": {
      "External id": 2481704,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9407
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615422.099, "dur": 2.609,
    "args": {
      "External id": 2481705,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9408
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615436.452, "dur": 19.039,
    "args": {
      "External id": 2481706,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9409
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615456.739, "dur": 12.065,
    "args": {
      "External id": 2481707,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9410
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615471.462, "dur": 13.225,
    "args": {
      "External id": 2481708,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9411
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615486.068, "dur": 11.493,
    "args": {
      "External id": 2481709,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9412
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615499.080, "dur": 20.949,
    "args": {
      "External id": 2481710,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9413
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615500.734, "dur": 1.361,
    "args": {
      "External id": 2481711,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9414
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615505.348, "dur": 2.296,
    "args": {
      "External id": 2481712,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9415
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615521.431, "dur": 27.758,
    "args": {
      "External id": 2481713,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9416
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615551.096, "dur": 11.442,
    "args": {
      "External id": 2481714,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9417
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615572.042, "dur": 2.183,
    "args": {
      "External id": 2481715,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9418
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615583.891, "dur": 3.327,
    "args": {
      "External id": 2481716,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9419
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615586.028, "dur": 0.444,
    "args": {
      "External id": 2481717,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "40960"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9420
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615649.414, "dur": 52.750,
    "args": {
      "External id": 2481718,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9421
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615708.517, "dur": 4.753,
    "args": {
      "External id": 2481719,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9422
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615711.033, "dur": 0.924,
    "args": {
      "External id": 2481720,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "83886080"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9423
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615714.504, "dur": 23.070,
    "args": {
      "External id": 2481721,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9424
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615741.995, "dur": 5.141,
    "args": {
      "External id": 2481722,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9425
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615743.317, "dur": 3.171,
    "args": {
      "External id": 2481723,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9426
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615744.928, "dur": 1.399,
    "args": {
      "External id": 2481724,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9427
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615751.212, "dur": 39.409,
    "args": {
      "External id": 2481725,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9428
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615752.121, "dur": 37.913,
    "args": {
      "External id": 2481726,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9429
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615794.190, "dur": 13.514,
    "args": {
      "External id": 2481727,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9430
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615813.108, "dur": 3.847,
    "args": {
      "External id": 2481728,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9431
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615815.179, "dur": 0.877,
    "args": {
      "External id": 2481729,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "100663296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9432
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615820.472, "dur": 47.653,
    "args": {
      "External id": 2481730,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9433
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615821.262, "dur": 4.921,
    "args": {
      "External id": 2481731,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9434
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615823.847, "dur": 1.832,
    "args": {
      "External id": 2481732,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9435
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615825.003, "dur": 0.524,
    "args": {
      "External id": 2481733,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9436
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615826.915, "dur": 40.871,
    "args": {
      "External id": 2481734,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9437
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615827.330, "dur": 39.981,
    "args": {
      "External id": 2481735,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9438
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615872.257, "dur": 3.619,
    "args": {
      "External id": 2481736,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9439
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615874.158, "dur": 0.713,
    "args": {
      "External id": 2481737,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "49152"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9440
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615880.907, "dur": 1.216,
    "args": {
      "External id": 2481738,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9441
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615890.063, "dur": 6.011,
    "args": {
      "External id": 2481739,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9442
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615891.853, "dur": 3.929,
    "args": {
      "External id": 2481740,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9443
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615972.212, "dur": 215.475,
    "args": {
      "External id": 2481741,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9444
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615976.023, "dur": 1.912,
    "args": {
      "External id": 2481742,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9445
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615979.251, "dur": 207.684,
    "args": {
      "External id": 2481743,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9446
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336754, "tid": 1336754,
    "ts": 1514656615980.231, "dur": 34.118,
    "args": {
      "External id": 2481744,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9447
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616016.660, "dur": 23.488,
    "args": {
      "External id": 2481745,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9448
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616043.322, "dur": 4.814,
    "args": {
      "External id": 2481746,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9449
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616046.805, "dur": 1.004,
    "args": {
      "External id": 2481747,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9450
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616055.788, "dur": 20.206,
    "args": {
      "External id": 2481748,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9451
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616056.833, "dur": 1.719,
    "args": {
      "External id": 2481749,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9452
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616059.772, "dur": 15.961,
    "args": {
      "External id": 2481750,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9453
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616062.126, "dur": 2.501,
    "args": {
      "External id": 2481751,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9454
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616077.207, "dur": 18.893,
    "args": {
      "External id": 2481752,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9455
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616097.447, "dur": 14.185,
    "args": {
      "External id": 2481753,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9456
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616114.277, "dur": 12.701,
    "args": {
      "External id": 2481754,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9457
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616128.221, "dur": 11.409,
    "args": {
      "External id": 2481755,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9458
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616143.190, "dur": 17.564,
    "args": {
      "External id": 2481756,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9459
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616145.194, "dur": 1.433,
    "args": {
      "External id": 2481757,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9460
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616148.330, "dur": 0.566,
    "args": {
      "External id": 2481758,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9461
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616162.098, "dur": 11.380,
    "args": {
      "External id": 2481759,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9462
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616174.595, "dur": 11.317,
    "args": {
      "External id": 2481760,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9463
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616195.310, "dur": 2.255,
    "args": {
      "External id": 2481761,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9464
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616205.815, "dur": 3.428,
    "args": {
      "External id": 2481762,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9465
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616208.062, "dur": 0.380,
    "args": {
      "External id": 2481763,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "49152"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9466
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616273.924, "dur": 52.864,
    "args": {
      "External id": 2481764,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9467
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616331.524, "dur": 4.451,
    "args": {
      "External id": 2481765,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9468
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616333.859, "dur": 1.120,
    "args": {
      "External id": 2481766,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "100663296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9469
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616340.848, "dur": 21.938,
    "args": {
      "External id": 2481767,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9470
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616367.181, "dur": 11.227,
    "args": {
      "External id": 2481768,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9471
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616368.709, "dur": 8.926,
    "args": {
      "External id": 2481769,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9472
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616374.959, "dur": 2.473,
    "args": {
      "External id": 2481770,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9473
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616380.884, "dur": 41.566,
    "args": {
      "External id": 2481771,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9474
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616382.062, "dur": 39.761,
    "args": {
      "External id": 2481772,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9475
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616425.958, "dur": 13.397,
    "args": {
      "External id": 2481773,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9476
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616444.962, "dur": 3.454,
    "args": {
      "External id": 2481774,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9477
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616446.895, "dur": 0.669,
    "args": {
      "External id": 2481775,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "117440512"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9478
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616453.522, "dur": 45.912,
    "args": {
      "External id": 2481776,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9479
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616454.425, "dur": 3.594,
    "args": {
      "External id": 2481777,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9480
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616455.355, "dur": 2.176,
    "args": {
      "External id": 2481778,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9481
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616456.595, "dur": 0.743,
    "args": {
      "External id": 2481779,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9482
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616458.581, "dur": 40.504,
    "args": {
      "External id": 2481780,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9483
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616459.155, "dur": 39.359,
    "args": {
      "External id": 2481781,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9484
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616504.787, "dur": 3.586,
    "args": {
      "External id": 2481782,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9485
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616506.562, "dur": 0.690,
    "args": {
      "External id": 2481783,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "57344"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9486
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616513.032, "dur": 1.243,
    "args": {
      "External id": 2481784,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9487
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616520.503, "dur": 5.783,
    "args": {
      "External id": 2481785,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9488
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616521.996, "dur": 4.016,
    "args": {
      "External id": 2481786,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9489
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616621.173, "dur": 167.844,
    "args": {
      "External id": 2481787,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9490
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616623.266, "dur": 2.845,
    "args": {
      "External id": 2481788,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9491
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616629.422, "dur": 159.103,
    "args": {
      "External id": 2481789,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9492
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616630.526, "dur": 0.369,
    "args": {
      "External id": 2481790,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9493
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616631.776, "dur": 20.332,
    "args": {
      "External id": 2481791,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9494
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616653.326, "dur": 4.099,
    "args": {
      "External id": 2481792,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9495
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616654.888, "dur": 2.260,
    "args": {
      "External id": 2481793,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9496
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616658.206, "dur": 20.573,
    "args": {
      "External id": 2481794,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9497
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616659.345, "dur": 1.462,
    "args": {
      "External id": 2481795,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9498
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616661.855, "dur": 16.635,
    "args": {
      "External id": 2481796,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9499
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616665.614, "dur": 2.354,
    "args": {
      "External id": 2481797,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9500
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616680.010, "dur": 19.326,
    "args": {
      "External id": 2481798,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9501
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616700.754, "dur": 11.803,
    "args": {
      "External id": 2481799,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9502
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616714.775, "dur": 12.691,
    "args": {
      "External id": 2481800,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9503
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616728.682, "dur": 11.760,
    "args": {
      "External id": 2481801,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9504
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616741.793, "dur": 19.308,
    "args": {
      "External id": 2481802,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9505
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616743.581, "dur": 1.218,
    "args": {
      "External id": 2481803,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9506
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616746.394, "dur": 2.406,
    "args": {
      "External id": 2481804,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9507
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616764.406, "dur": 10.933,
    "args": {
      "External id": 2481805,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9508
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616776.112, "dur": 11.603,
    "args": {
      "External id": 2481806,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9509
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616795.483, "dur": 1.755,
    "args": {
      "External id": 2481807,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9510
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616804.613, "dur": 3.318,
    "args": {
      "External id": 2481808,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9511
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616806.653, "dur": 0.412,
    "args": {
      "External id": 2481809,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "57344"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9512
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616862.656, "dur": 49.982,
    "args": {
      "External id": 2481810,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9513
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616917.256, "dur": 4.420,
    "args": {
      "External id": 2481811,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9514
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616919.600, "dur": 1.037,
    "args": {
      "External id": 2481812,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "117440512"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9515
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616924.519, "dur": 21.778,
    "args": {
      "External id": 2481813,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9516
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616950.336, "dur": 5.118,
    "args": {
      "External id": 2481814,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9517
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616951.599, "dur": 3.297,
    "args": {
      "External id": 2481815,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9518
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616953.402, "dur": 1.324,
    "args": {
      "External id": 2481816,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9519
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616957.408, "dur": 79.145,
    "args": {
      "External id": 2481817,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9520
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656616958.541, "dur": 76.712,
    "args": {
      "External id": 2481818,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9521
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617042.099, "dur": 15.713,
    "args": {
      "External id": 2481819,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9522
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617064.605, "dur": 24.225,
    "args": {
      "External id": 2481820,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", ""], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 9523
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617067.261, "dur": 21.105,
    "args": {
      "External id": 2481821,"Record function id": 0, "Concrete Inputs": ["", "[]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9524
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617072.648, "dur": 1.621,
    "args": {
      "External id": 2481822,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 9525
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617094.400, "dur": 25.064,
    "args": {
      "External id": 2481823,"Record function id": 0, "Concrete Inputs": ["", "", "15", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], []], "Ev Idx": 9526
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617096.296, "dur": 22.964,
    "args": {
      "External id": 2481824,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], [], []], "Ev Idx": 9527
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617100.561, "dur": 4.233,
    "args": {
      "External id": 2481825,"Record function id": 0, "Concrete Inputs": ["[32000, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9528
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617105.888, "dur": 12.897,
    "args": {
      "External id": 2481826,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "float", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9529
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617133.144, "dur": 4.746,
    "args": {
      "External id": 2481827,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 9530
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617134.994, "dur": 2.658,
    "args": {
      "External id": 2481828,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 9531
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617138.933, "dur": 3.122,
    "args": {
      "External id": 2481829,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9532
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617141.210, "dur": 0.704,
    "args": {
      "External id": 2481830,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9533
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617179.976, "dur": 20.878,
    "args": {
      "External id": 2481831,"Sequence number": 24740632, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[1], [], []], "Ev Idx": 9534
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617203.074, "dur": 13.136,
    "args": {
      "External id": 2481832,"Sequence number": 24740633, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[1], [], []], "Ev Idx": 9535
    }
  },
  {
    "ph": "s", "id": 16, "pid": 1336754, "tid": 1336754, "ts": 1514656617203.074,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617221.972, "dur": 6.106,
    "args": {
      "External id": 2481833,"Sequence number": 24740634, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], [], []], "Input Dims": [[16, 4, 4096], [], [], [], []], "Ev Idx": 9536
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617225.440, "dur": 1.291,
    "args": {
      "External id": 2481834,"Record function id": 0, "Concrete Inputs": ["", "[16, 4, 4096]", "[8192, 1, 1]", "1"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 4, 4096], [], [], []], "Ev Idx": 9537
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617230.494, "dur": 7.638,
    "args": {
      "External id": 2481835,"Sequence number": 24740634, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "2"], "Input type": ["long int", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], []], "Input Dims": [[16, 4, 4096], [], []], "Ev Idx": 9538
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617236.031, "dur": 0.832,
    "args": {
      "External id": 2481836,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096]", "[8192, 1]", "3"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 4, 4096], [], [], []], "Ev Idx": 9539
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617239.287, "dur": 2.574,
    "args": {
      "External id": 2481837,"Sequence number": 24740634, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "0", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], []], "Ev Idx": 9540
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617240.629, "dur": 0.692,
    "args": {
      "External id": 2481838,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096]", "[8192, 1]", "3"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1], [], [], []], "Input Dims": [[16, 4096], [], [], []], "Ev Idx": 9541
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617245.640, "dur": 5.549,
    "args": {
      "External id": 2481839,"Sequence number": 24740634, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 9542
    }
  },
  {
    "ph": "s", "id": 15, "pid": 1336754, "tid": 1336754, "ts": 1514656617245.640,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617248.931, "dur": 1.019,
    "args": {
      "External id": 2481840,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 9543
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617252.102, "dur": 5.967,
    "args": {
      "External id": 2481841,"Sequence number": 24740635, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 9544
    }
  },
  {
    "ph": "s", "id": 14, "pid": 1336754, "tid": 1336754, "ts": 1514656617252.102,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617256.395, "dur": 0.921,
    "args": {
      "External id": 2481842,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 9545
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617259.015, "dur": 4.622,
    "args": {
      "External id": 2481843,"Sequence number": 24740636, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "2", "2"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], []], "Input Dims": [[16, 4096, 4, 2048], [], []], "Ev Idx": 9546
    }
  },
  {
    "ph": "s", "id": 13, "pid": 1336754, "tid": 1336754, "ts": 1514656617259.015,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617261.913, "dur": 0.894,
    "args": {
      "External id": 2481844,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "4096"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 9547
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617264.800, "dur": 4.368,
    "args": {
      "External id": 2481845,"Sequence number": 24740637, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "2", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 1], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], []], "Ev Idx": 9548
    }
  },
  {
    "ph": "s", "id": 12, "pid": 1336754, "tid": 1336754, "ts": 1514656617264.800,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617267.534, "dur": 0.924,
    "args": {
      "External id": 2481846,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "4096"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 1], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], []], "Ev Idx": 9549
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::contiguous", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617272.922, "dur": 31.436,
    "args": {
      "External id": 2481847,"Sequence number": 24740638, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["long int", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 9550
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617274.387, "dur": 29.768,
    "args": {
      "External id": 2481848,"Sequence number": 24740638, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["long int", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 9551
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617276.536, "dur": 7.788,
    "args": {
      "External id": 2481849,"Record function id": 0, "Concrete Inputs": ["", "4", "0", "", "", "0"], "Input type": ["long int", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[8192, 1], [], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], [], []], "Ev Idx": 9552
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617280.289, "dur": 3.481,
    "args": {
      "External id": 2481850,"Record function id": 0, "Concrete Inputs": ["[16, 4096]", "4", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9553
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617285.039, "dur": 18.547,
    "args": {
      "External id": 2481851,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[4096, 1], [8192, 1], []], "Input Dims": [[16, 4096], [16, 4096], []], "Ev Idx": 9554
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617328.645, "dur": 5.558,
    "args": {
      "External id": 2481852,"Sequence number": 24740638, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 9555
    }
  },
  {
    "ph": "s", "id": 11, "pid": 1336754, "tid": 1336754, "ts": 1514656617328.645,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617336.466, "dur": 1.194,
    "args": {
      "External id": 2481853,"Sequence number": 24740639, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["long int", "ScalarList"], "Input Strides": [[4096, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 9556
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FusedLinearCrossEntropyFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617367.874, "dur": 44056.970,
    "args": {
      "External id": 2481854,"Sequence number": 24740639, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "-100", "0.", "1.", "8"], "Input type": ["c10::BFloat16", "long int", "c10::BFloat16", "", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1], [1], [2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536], [32000, 2048], [], [], [], [], []], "Ev Idx": 9557
    }
  },
  {
    "ph": "s", "id": 10, "pid": 1336754, "tid": 1336754, "ts": 1514656617367.874,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::contiguous", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617381.662, "dur": 25.379,
    "args": {
      "External id": 2481855,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 9558
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617383.787, "dur": 23.019,
    "args": {
      "External id": 2481856,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 9559
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617385.081, "dur": 5.686,
    "args": {
      "External id": 2481857,"Record function id": 0, "Concrete Inputs": ["", "15", "0", "", "", "0"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[8192, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 9560
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617386.470, "dur": 3.859,
    "args": {
      "External id": 2481858,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9561
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617391.391, "dur": 15.085,
    "args": {
      "External id": 2481859,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [8192, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 9562
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617421.708, "dur": 22.392,
    "args": {
      "External id": 2481860,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 9563
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617422.614, "dur": 5.510,
    "args": {
      "External id": 2481861,"Record function id": 0, "Concrete Inputs": ["", "15", "0", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 9564
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617424.366, "dur": 3.416,
    "args": {
      "External id": 2481862,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9565
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617429.336, "dur": 14.552,
    "args": {
      "External id": 2481863,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 9566
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617430.972, "dur": 12.301,
    "args": {
      "External id": 2481864,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 9567
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617447.404, "dur": 18.894,
    "args": {
      "External id": 2481865,"Record function id": 0, "Concrete Inputs": ["", "6", "", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], []], "Ev Idx": 9568
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617448.256, "dur": 4.224,
    "args": {
      "External id": 2481866,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], []], "Ev Idx": 9569
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617449.317, "dur": 2.953,
    "args": {
      "External id": 2481867,"Record function id": 0, "Concrete Inputs": ["[32000, 2048]", "[2048, 1]", "6", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9570
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617454.561, "dur": 11.509,
    "args": {
      "External id": 2481868,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9571
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617455.304, "dur": 10.438,
    "args": {
      "External id": 2481869,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[2048, 1], []], "Input Dims": [[32000, 2048], []], "Ev Idx": 9572
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617471.912, "dur": 17.334,
    "args": {
      "External id": 2481870,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False"], "Input type": ["ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 9573
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617475.136, "dur": 2.578,
    "args": {
      "External id": 2481871,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9574
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617478.307, "dur": 10.682,
    "args": {
      "External id": 2481872,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[65536]], "Ev Idx": 9575
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617479.017, "dur": 9.697,
    "args": {
      "External id": 2481873,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 9576
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::ne", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617493.877, "dur": 20.821,
    "args": {
      "External id": 2481874,"Record function id": 0, "Concrete Inputs": ["", "-100"], "Input type": ["long int", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 9577
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617517.314, "dur": 67.819,
    "args": {
      "External id": 2481875,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["bool", ""], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 9578
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617520.811, "dur": 63.711,
    "args": {
      "External id": 2481876,"Record function id": 0, "Concrete Inputs": ["", "[]", "False", ""], "Input type": ["bool", "ScalarList", "Scalar", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9579
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617524.758, "dur": 0.991,
    "args": {
      "External id": 2481877,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[0]", ""], "Input type": ["long int", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 9580
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617542.773, "dur": 26.846,
    "args": {
      "External id": 2481878,"Record function id": 0, "Concrete Inputs": ["", "4", "False", "False", ""], "Input type": ["bool", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9581
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617544.878, "dur": 24.545,
    "args": {
      "External id": 2481879,"Record function id": 0, "Concrete Inputs": ["", "4", "", "", "", "False", ""], "Input type": ["bool", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[65536], [], [], [], [], [], []], "Ev Idx": 9582
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617549.556, "dur": 4.262,
    "args": {
      "External id": 2481880,"Record function id": 0, "Concrete Inputs": ["[65536]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9583
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617554.792, "dur": 14.331,
    "args": {
      "External id": 2481881,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["long int", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[65536], [65536], []], "Ev Idx": 9584
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::item", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617589.268, "dur": 38317.047,
    "args": {
      "External id": 2481882,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["long int"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 9585
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_local_scalar_dense", "pid": 1336754, "tid": 1336754,
    "ts": 1514656617590.679, "dur": 38314.699,
    "args": {
      "External id": 2481883,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["long int"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 9586
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656655916.082, "dur": 6.323,
    "args": {
      "External id": 2481884,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9587
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656655919.692, "dur": 1.065,
    "args": {
      "External id": 2481885,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9588
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656655928.932, "dur": 126.133,
    "args": {
      "External id": 2481886,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9589
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656655930.467, "dur": 5.875,
    "args": {
      "External id": 2481887,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9590
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656655932.491, "dur": 3.131,
    "args": {
      "External id": 2481888,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9591
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656655934.346, "dur": 1.022,
    "args": {
      "External id": 2481889,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9592
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656655937.637, "dur": 116.885,
    "args": {
      "External id": 2481890,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9593
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656655939.499, "dur": 113.626,
    "args": {
      "External id": 2481891,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9594
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656060.614, "dur": 5.479,
    "args": {
      "External id": 2481892,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9595
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656063.520, "dur": 0.924,
    "args": {
      "External id": 2481893,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "0"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9596
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656075.231, "dur": 2.253,
    "args": {
      "External id": 2481894,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9597
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656086.506, "dur": 6.335,
    "args": {
      "External id": 2481895,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9598
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656088.427, "dur": 4.132,
    "args": {
      "External id": 2481896,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9599
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656219.344, "dur": 193.769,
    "args": {
      "External id": 2481897,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9600
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656222.120, "dur": 2.655,
    "args": {
      "External id": 2481898,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9601
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656229.961, "dur": 182.538,
    "args": {
      "External id": 2481899,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9602
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656231.661, "dur": 0.423,
    "args": {
      "External id": 2481900,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9603
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656233.329, "dur": 24.983,
    "args": {
      "External id": 2481901,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9604
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656260.216, "dur": 5.783,
    "args": {
      "External id": 2481902,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9605
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656262.453, "dur": 3.175,
    "args": {
      "External id": 2481903,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9606
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656266.975, "dur": 22.602,
    "args": {
      "External id": 2481904,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9607
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656268.128, "dur": 1.330,
    "args": {
      "External id": 2481905,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9608
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656270.761, "dur": 18.483,
    "args": {
      "External id": 2481906,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9609
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656274.328, "dur": 3.092,
    "args": {
      "External id": 2481907,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9610
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656293.009, "dur": 22.000,
    "args": {
      "External id": 2481908,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9611
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656316.735, "dur": 13.715,
    "args": {
      "External id": 2481909,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9612
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656333.301, "dur": 15.555,
    "args": {
      "External id": 2481910,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9613
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656350.282, "dur": 12.928,
    "args": {
      "External id": 2481911,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9614
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656365.037, "dur": 18.761,
    "args": {
      "External id": 2481912,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9615
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656367.037, "dur": 1.382,
    "args": {
      "External id": 2481913,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9616
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656370.679, "dur": 0.716,
    "args": {
      "External id": 2481914,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9617
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656387.528, "dur": 11.681,
    "args": {
      "External id": 2481915,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9618
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656400.837, "dur": 10.606,
    "args": {
      "External id": 2481916,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9619
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656419.887, "dur": 1.983,
    "args": {
      "External id": 2481917,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9620
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656428.657, "dur": 3.816,
    "args": {
      "External id": 2481918,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9621
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656430.714, "dur": 0.682,
    "args": {
      "External id": 2481919,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9622
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656504.579, "dur": 84.819,
    "args": {
      "External id": 2481920,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9623
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656596.714, "dur": 8.662,
    "args": {
      "External id": 2481921,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9624
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656600.101, "dur": 2.497,
    "args": {
      "External id": 2481922,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9625
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656608.614, "dur": 27.810,
    "args": {
      "External id": 2481923,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9626
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656642.287, "dur": 5.867,
    "args": {
      "External id": 2481924,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9627
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656644.159, "dur": 3.260,
    "args": {
      "External id": 2481925,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9628
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656646.163, "dur": 0.985,
    "args": {
      "External id": 2481926,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9629
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656651.528, "dur": 47.376,
    "args": {
      "External id": 2481927,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9630
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656652.736, "dur": 45.531,
    "args": {
      "External id": 2481928,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9631
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656703.788, "dur": 16.047,
    "args": {
      "External id": 2481929,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9632
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656727.922, "dur": 3.355,
    "args": {
      "External id": 2481930,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9633
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656729.654, "dur": 0.610,
    "args": {
      "External id": 2481931,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "16777216"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9634
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656735.241, "dur": 51.299,
    "args": {
      "External id": 2481932,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9635
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656736.285, "dur": 3.877,
    "args": {
      "External id": 2481933,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9636
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656737.043, "dur": 2.504,
    "args": {
      "External id": 2481934,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9637
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656738.477, "dur": 0.919,
    "args": {
      "External id": 2481935,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9638
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656742.645, "dur": 43.489,
    "args": {
      "External id": 2481936,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9639
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656743.331, "dur": 42.404,
    "args": {
      "External id": 2481937,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9640
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656790.726, "dur": 3.866,
    "args": {
      "External id": 2481938,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9641
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656792.582, "dur": 0.676,
    "args": {
      "External id": 2481939,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "8192"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9642
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656801.127, "dur": 1.782,
    "args": {
      "External id": 2481940,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9643
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656810.001, "dur": 7.995,
    "args": {
      "External id": 2481941,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9644
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656813.702, "dur": 4.021,
    "args": {
      "External id": 2481942,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9645
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656913.151, "dur": 223.914,
    "args": {
      "External id": 2481943,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9646
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656915.520, "dur": 3.511,
    "args": {
      "External id": 2481944,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9647
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656920.413, "dur": 216.171,
    "args": {
      "External id": 2481945,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9648
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656921.526, "dur": 0.395,
    "args": {
      "External id": 2481946,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9649
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656923.338, "dur": 20.259,
    "args": {
      "External id": 2481947,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9650
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656944.981, "dur": 5.188,
    "args": {
      "External id": 2481948,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9651
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656949.276, "dur": 0.592,
    "args": {
      "External id": 2481949,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9652
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656953.260, "dur": 62.877,
    "args": {
      "External id": 2481950,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9653
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656954.034, "dur": 1.632,
    "args": {
      "External id": 2481951,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9654
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656956.884, "dur": 58.491,
    "args": {
      "External id": 2481952,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9655
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656656959.201, "dur": 2.706,
    "args": {
      "External id": 2481953,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9656
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657018.268, "dur": 23.282,
    "args": {
      "External id": 2481954,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9657
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657043.017, "dur": 13.146,
    "args": {
      "External id": 2481955,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9658
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657058.836, "dur": 13.459,
    "args": {
      "External id": 2481956,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9659
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657073.762, "dur": 11.820,
    "args": {
      "External id": 2481957,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9660
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657087.439, "dur": 22.756,
    "args": {
      "External id": 2481958,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9661
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657091.262, "dur": 1.586,
    "args": {
      "External id": 2481959,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9662
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657094.863, "dur": 2.064,
    "args": {
      "External id": 2481960,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9663
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657111.830, "dur": 11.864,
    "args": {
      "External id": 2481961,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9664
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657124.843, "dur": 10.659,
    "args": {
      "External id": 2481962,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9665
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657145.703, "dur": 2.542,
    "args": {
      "External id": 2481963,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9666
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657158.268, "dur": 3.855,
    "args": {
      "External id": 2481964,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9667
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657160.857, "dur": 0.361,
    "args": {
      "External id": 2481965,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "8192"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9668
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657233.969, "dur": 57.043,
    "args": {
      "External id": 2481966,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9669
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657298.135, "dur": 4.571,
    "args": {
      "External id": 2481967,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9670
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657300.739, "dur": 0.860,
    "args": {
      "External id": 2481968,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "16777216"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9671
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657303.990, "dur": 24.620,
    "args": {
      "External id": 2481969,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9672
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657333.258, "dur": 5.494,
    "args": {
      "External id": 2481970,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9673
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657334.600, "dur": 3.402,
    "args": {
      "External id": 2481971,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9674
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657336.506, "dur": 1.285,
    "args": {
      "External id": 2481972,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9675
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657343.055, "dur": 42.899,
    "args": {
      "External id": 2481973,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9676
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657343.988, "dur": 41.211,
    "args": {
      "External id": 2481974,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9677
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657389.911, "dur": 13.735,
    "args": {
      "External id": 2481975,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9678
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657409.497, "dur": 3.517,
    "args": {
      "External id": 2481976,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9679
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657411.582, "dur": 0.632,
    "args": {
      "External id": 2481977,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "33554432"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9680
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657416.655, "dur": 48.939,
    "args": {
      "External id": 2481978,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9681
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657417.454, "dur": 5.090,
    "args": {
      "External id": 2481979,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9682
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657420.143, "dur": 1.912,
    "args": {
      "External id": 2481980,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9683
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657421.330, "dur": 0.567,
    "args": {
      "External id": 2481981,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9684
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657423.176, "dur": 42.035,
    "args": {
      "External id": 2481982,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9685
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657423.619, "dur": 40.963,
    "args": {
      "External id": 2481983,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9686
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657483.327, "dur": 3.992,
    "args": {
      "External id": 2481984,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9687
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657485.589, "dur": 0.611,
    "args": {
      "External id": 2481985,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "16384"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9688
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657492.982, "dur": 1.351,
    "args": {
      "External id": 2481986,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9689
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657503.509, "dur": 7.573,
    "args": {
      "External id": 2481987,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9690
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657505.188, "dur": 5.619,
    "args": {
      "External id": 2481988,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9691
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657611.282, "dur": 175.831,
    "args": {
      "External id": 2481989,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9692
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657613.502, "dur": 2.778,
    "args": {
      "External id": 2481990,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9693
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657617.905, "dur": 168.781,
    "args": {
      "External id": 2481991,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9694
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657619.388, "dur": 0.248,
    "args": {
      "External id": 2481992,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9695
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657620.608, "dur": 23.274,
    "args": {
      "External id": 2481993,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9696
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657645.593, "dur": 3.013,
    "args": {
      "External id": 2481994,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9697
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657647.458, "dur": 0.837,
    "args": {
      "External id": 2481995,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9698
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657651.075, "dur": 26.517,
    "args": {
      "External id": 2481996,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9699
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657652.062, "dur": 1.439,
    "args": {
      "External id": 2481997,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9700
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657654.528, "dur": 22.777,
    "args": {
      "External id": 2481998,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9701
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657658.585, "dur": 2.586,
    "args": {
      "External id": 2481999,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9702
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657678.892, "dur": 19.072,
    "args": {
      "External id": 2482000,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9703
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657699.425, "dur": 12.454,
    "args": {
      "External id": 2482001,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9704
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657714.455, "dur": 12.961,
    "args": {
      "External id": 2482002,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9705
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657728.668, "dur": 11.134,
    "args": {
      "External id": 2482003,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9706
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657741.619, "dur": 19.586,
    "args": {
      "External id": 2482004,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9707
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657744.575, "dur": 1.332,
    "args": {
      "External id": 2482005,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9708
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657747.935, "dur": 0.641,
    "args": {
      "External id": 2482006,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9709
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657762.836, "dur": 10.978,
    "args": {
      "External id": 2482007,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9710
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657774.741, "dur": 11.034,
    "args": {
      "External id": 2482008,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9711
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657794.274, "dur": 2.070,
    "args": {
      "External id": 2482009,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9712
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657805.183, "dur": 3.671,
    "args": {
      "External id": 2482010,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9713
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657807.507, "dur": 0.592,
    "args": {
      "External id": 2482011,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "16384"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9714
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657870.966, "dur": 54.792,
    "args": {
      "External id": 2482012,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9715
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657930.445, "dur": 4.086,
    "args": {
      "External id": 2482013,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9716
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657932.620, "dur": 0.781,
    "args": {
      "External id": 2482014,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "33554432"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9717
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657935.997, "dur": 32.359,
    "args": {
      "External id": 2482015,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9718
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657973.112, "dur": 7.492,
    "args": {
      "External id": 2482016,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9719
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657974.513, "dur": 5.453,
    "args": {
      "External id": 2482017,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9720
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656657978.584, "dur": 1.210,
    "args": {
      "External id": 2482018,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9721
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658020.784, "dur": 85.966,
    "args": {
      "External id": 2482019,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9722
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658021.933, "dur": 84.016,
    "args": {
      "External id": 2482020,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9723
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658111.164, "dur": 29.784,
    "args": {
      "External id": 2482021,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9724
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658147.631, "dur": 4.807,
    "args": {
      "External id": 2482022,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9725
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658150.222, "dur": 1.105,
    "args": {
      "External id": 2482023,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "50331648"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9726
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658156.321, "dur": 55.332,
    "args": {
      "External id": 2482024,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9727
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658159.052, "dur": 3.906,
    "args": {
      "External id": 2482025,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9728
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658160.165, "dur": 2.250,
    "args": {
      "External id": 2482026,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9729
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658161.588, "dur": 0.659,
    "args": {
      "External id": 2482027,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9730
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658163.578, "dur": 47.648,
    "args": {
      "External id": 2482028,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9731
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658164.021, "dur": 46.552,
    "args": {
      "External id": 2482029,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9732
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658216.004, "dur": 5.196,
    "args": {
      "External id": 2482030,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9733
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658217.603, "dur": 2.295,
    "args": {
      "External id": 2482031,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "24576"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9734
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658229.058, "dur": 1.360,
    "args": {
      "External id": 2482032,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9735
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658237.468, "dur": 6.411,
    "args": {
      "External id": 2482033,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9736
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658239.420, "dur": 4.211,
    "args": {
      "External id": 2482034,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9737
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658326.773, "dur": 167.812,
    "args": {
      "External id": 2482035,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9738
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658329.127, "dur": 2.118,
    "args": {
      "External id": 2482036,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9739
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658334.490, "dur": 159.655,
    "args": {
      "External id": 2482037,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9740
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658335.832, "dur": 0.241,
    "args": {
      "External id": 2482038,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9741
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658337.199, "dur": 21.006,
    "args": {
      "External id": 2482039,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9742
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658359.778, "dur": 3.051,
    "args": {
      "External id": 2482040,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9743
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658361.851, "dur": 0.688,
    "args": {
      "External id": 2482041,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9744
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658363.612, "dur": 21.167,
    "args": {
      "External id": 2482042,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9745
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658364.533, "dur": 1.341,
    "args": {
      "External id": 2482043,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9746
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658367.187, "dur": 17.257,
    "args": {
      "External id": 2482044,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9747
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658371.178, "dur": 2.501,
    "args": {
      "External id": 2482045,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9748
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658387.869, "dur": 18.510,
    "args": {
      "External id": 2482046,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9749
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658407.736, "dur": 12.172,
    "args": {
      "External id": 2482047,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9750
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658422.583, "dur": 12.385,
    "args": {
      "External id": 2482048,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9751
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658436.315, "dur": 11.330,
    "args": {
      "External id": 2482049,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9752
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658449.461, "dur": 18.346,
    "args": {
      "External id": 2482050,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9753
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658451.194, "dur": 1.183,
    "args": {
      "External id": 2482051,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9754
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658454.352, "dur": 0.824,
    "args": {
      "External id": 2482052,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9755
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658469.194, "dur": 11.713,
    "args": {
      "External id": 2482053,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9756
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658483.841, "dur": 9.230,
    "args": {
      "External id": 2482054,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9757
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658500.774, "dur": 1.612,
    "args": {
      "External id": 2482055,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9758
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658510.145, "dur": 3.230,
    "args": {
      "External id": 2482056,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9759
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658512.183, "dur": 0.450,
    "args": {
      "External id": 2482057,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "24576"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9760
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658592.634, "dur": 53.014,
    "args": {
      "External id": 2482058,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9761
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658650.864, "dur": 5.874,
    "args": {
      "External id": 2482059,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9762
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658653.629, "dur": 1.551,
    "args": {
      "External id": 2482060,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "50331648"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9763
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658658.244, "dur": 23.698,
    "args": {
      "External id": 2482061,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9764
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658688.102, "dur": 5.650,
    "args": {
      "External id": 2482062,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9765
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658689.571, "dur": 3.428,
    "args": {
      "External id": 2482063,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9766
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658691.650, "dur": 1.163,
    "args": {
      "External id": 2482064,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9767
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658696.224, "dur": 40.709,
    "args": {
      "External id": 2482065,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9768
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658697.490, "dur": 38.754,
    "args": {
      "External id": 2482066,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9769
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658740.796, "dur": 13.666,
    "args": {
      "External id": 2482067,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9770
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658759.897, "dur": 4.997,
    "args": {
      "External id": 2482068,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9771
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658763.317, "dur": 0.702,
    "args": {
      "External id": 2482069,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "67108864"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9772
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658768.345, "dur": 49.878,
    "args": {
      "External id": 2482070,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9773
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658768.967, "dur": 5.701,
    "args": {
      "External id": 2482071,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9774
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658769.814, "dur": 4.279,
    "args": {
      "External id": 2482072,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9775
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658772.981, "dur": 0.987,
    "args": {
      "External id": 2482073,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9776
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658775.381, "dur": 42.403,
    "args": {
      "External id": 2482074,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9777
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658777.907, "dur": 39.289,
    "args": {
      "External id": 2482075,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9778
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658828.671, "dur": 4.557,
    "args": {
      "External id": 2482076,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9779
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658830.917, "dur": 1.005,
    "args": {
      "External id": 2482077,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "32768"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9780
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658839.119, "dur": 1.327,
    "args": {
      "External id": 2482078,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9781
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658847.457, "dur": 7.606,
    "args": {
      "External id": 2482079,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9782
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658850.808, "dur": 3.937,
    "args": {
      "External id": 2482080,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9783
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658935.873, "dur": 210.167,
    "args": {
      "External id": 2482081,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9784
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658939.581, "dur": 2.041,
    "args": {
      "External id": 2482082,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9785
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658942.727, "dur": 202.530,
    "args": {
      "External id": 2482083,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9786
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658943.589, "dur": 0.349,
    "args": {
      "External id": 2482084,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9787
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658945.073, "dur": 18.854,
    "args": {
      "External id": 2482085,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9788
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658965.669, "dur": 4.749,
    "args": {
      "External id": 2482086,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9789
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658969.352, "dur": 0.796,
    "args": {
      "External id": 2482087,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9790
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658971.157, "dur": 58.276,
    "args": {
      "External id": 2482088,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9791
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658973.678, "dur": 1.570,
    "args": {
      "External id": 2482089,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9792
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658976.558, "dur": 52.534,
    "args": {
      "External id": 2482090,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9793
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656658979.171, "dur": 36.371,
    "args": {
      "External id": 2482091,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9794
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659031.292, "dur": 20.550,
    "args": {
      "External id": 2482092,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9795
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659053.215, "dur": 12.624,
    "args": {
      "External id": 2482093,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9796
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659068.592, "dur": 12.830,
    "args": {
      "External id": 2482094,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9797
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659082.774, "dur": 11.917,
    "args": {
      "External id": 2482095,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9798
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659096.599, "dur": 23.338,
    "args": {
      "External id": 2482096,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9799
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659100.409, "dur": 1.545,
    "args": {
      "External id": 2482097,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9800
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659106.028, "dur": 0.891,
    "args": {
      "External id": 2482098,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9801
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659121.241, "dur": 11.142,
    "args": {
      "External id": 2482099,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9802
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659133.325, "dur": 10.998,
    "args": {
      "External id": 2482100,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9803
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659153.968, "dur": 2.162,
    "args": {
      "External id": 2482101,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9804
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659164.984, "dur": 3.312,
    "args": {
      "External id": 2482102,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9805
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659167.092, "dur": 0.455,
    "args": {
      "External id": 2482103,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "32768"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9806
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659234.620, "dur": 59.826,
    "args": {
      "External id": 2482104,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9807
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659299.036, "dur": 6.524,
    "args": {
      "External id": 2482105,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9808
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659303.690, "dur": 0.858,
    "args": {
      "External id": 2482106,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "67108864"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9809
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659306.974, "dur": 24.846,
    "args": {
      "External id": 2482107,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9810
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659336.801, "dur": 5.098,
    "args": {
      "External id": 2482108,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9811
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659338.005, "dur": 3.152,
    "args": {
      "External id": 2482109,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9812
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659339.797, "dur": 1.112,
    "args": {
      "External id": 2482110,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9813
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659344.498, "dur": 44.879,
    "args": {
      "External id": 2482111,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9814
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659347.437, "dur": 41.327,
    "args": {
      "External id": 2482112,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9815
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659393.414, "dur": 13.773,
    "args": {
      "External id": 2482113,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9816
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659412.786, "dur": 3.583,
    "args": {
      "External id": 2482114,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9817
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659414.660, "dur": 0.896,
    "args": {
      "External id": 2482115,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "83886080"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9818
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659420.226, "dur": 56.807,
    "args": {
      "External id": 2482116,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9819
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659428.506, "dur": 4.776,
    "args": {
      "External id": 2482117,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9820
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659430.909, "dur": 1.804,
    "args": {
      "External id": 2482118,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9821
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659431.935, "dur": 0.648,
    "args": {
      "External id": 2482119,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9822
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659433.988, "dur": 42.621,
    "args": {
      "External id": 2482120,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9823
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659434.545, "dur": 41.550,
    "args": {
      "External id": 2482121,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9824
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659480.794, "dur": 4.076,
    "args": {
      "External id": 2482122,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9825
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659482.961, "dur": 0.616,
    "args": {
      "External id": 2482123,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "40960"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9826
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659491.470, "dur": 1.360,
    "args": {
      "External id": 2482124,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9827
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659499.901, "dur": 7.015,
    "args": {
      "External id": 2482125,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9828
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659501.654, "dur": 4.962,
    "args": {
      "External id": 2482126,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9829
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659609.143, "dur": 177.321,
    "args": {
      "External id": 2482127,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9830
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659611.238, "dur": 2.900,
    "args": {
      "External id": 2482128,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9831
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659615.756, "dur": 170.240,
    "args": {
      "External id": 2482129,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9832
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659619.269, "dur": 0.259,
    "args": {
      "External id": 2482130,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9833
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659620.556, "dur": 23.718,
    "args": {
      "External id": 2482131,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9834
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659646.102, "dur": 3.097,
    "args": {
      "External id": 2482132,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9835
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659647.946, "dur": 0.983,
    "args": {
      "External id": 2482133,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9836
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659650.151, "dur": 20.796,
    "args": {
      "External id": 2482134,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9837
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659651.195, "dur": 1.456,
    "args": {
      "External id": 2482135,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9838
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659653.668, "dur": 17.007,
    "args": {
      "External id": 2482136,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9839
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659657.422, "dur": 2.551,
    "args": {
      "External id": 2482137,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9840
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659672.126, "dur": 22.374,
    "args": {
      "External id": 2482138,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9841
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659695.806, "dur": 12.886,
    "args": {
      "External id": 2482139,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9842
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659711.040, "dur": 13.578,
    "args": {
      "External id": 2482140,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9843
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659727.546, "dur": 11.228,
    "args": {
      "External id": 2482141,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9844
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659740.326, "dur": 18.179,
    "args": {
      "External id": 2482142,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9845
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659741.911, "dur": 1.492,
    "args": {
      "External id": 2482143,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9846
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659745.151, "dur": 0.813,
    "args": {
      "External id": 2482144,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9847
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659759.953, "dur": 11.244,
    "args": {
      "External id": 2482145,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9848
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659772.229, "dur": 11.153,
    "args": {
      "External id": 2482146,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9849
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659793.152, "dur": 1.898,
    "args": {
      "External id": 2482147,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9850
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659803.529, "dur": 3.291,
    "args": {
      "External id": 2482148,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9851
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659805.664, "dur": 0.357,
    "args": {
      "External id": 2482149,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "40960"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9852
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659868.981, "dur": 53.585,
    "args": {
      "External id": 2482150,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9853
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659926.697, "dur": 4.919,
    "args": {
      "External id": 2482151,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9854
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659929.003, "dur": 1.291,
    "args": {
      "External id": 2482152,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "83886080"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9855
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659932.791, "dur": 21.500,
    "args": {
      "External id": 2482153,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9856
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659959.187, "dur": 7.285,
    "args": {
      "External id": 2482154,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9857
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659962.529, "dur": 3.289,
    "args": {
      "External id": 2482155,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9858
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659964.423, "dur": 1.210,
    "args": {
      "External id": 2482156,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9859
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659968.898, "dur": 80.691,
    "args": {
      "External id": 2482157,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9860
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656659969.922, "dur": 78.365,
    "args": {
      "External id": 2482158,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9861
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660055.243, "dur": 15.754,
    "args": {
      "External id": 2482159,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9862
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660077.183, "dur": 6.058,
    "args": {
      "External id": 2482160,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9863
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660081.142, "dur": 1.023,
    "args": {
      "External id": 2482161,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "100663296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9864
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660087.156, "dur": 49.918,
    "args": {
      "External id": 2482162,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9865
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660088.166, "dur": 3.823,
    "args": {
      "External id": 2482163,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9866
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660089.220, "dur": 2.231,
    "args": {
      "External id": 2482164,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9867
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660090.525, "dur": 0.754,
    "args": {
      "External id": 2482165,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9868
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660092.672, "dur": 43.920,
    "args": {
      "External id": 2482166,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9869
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660093.355, "dur": 42.450,
    "args": {
      "External id": 2482167,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9870
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660141.334, "dur": 5.341,
    "args": {
      "External id": 2482168,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9871
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660143.137, "dur": 2.340,
    "args": {
      "External id": 2482169,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "49152"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9872
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660154.315, "dur": 1.633,
    "args": {
      "External id": 2482170,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9873
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660163.187, "dur": 6.242,
    "args": {
      "External id": 2482171,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9874
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660164.906, "dur": 4.239,
    "args": {
      "External id": 2482172,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9875
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660246.261, "dur": 192.118,
    "args": {
      "External id": 2482173,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9876
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660248.433, "dur": 2.172,
    "args": {
      "External id": 2482174,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9877
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660253.568, "dur": 184.414,
    "args": {
      "External id": 2482175,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9878
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660254.664, "dur": 0.236,
    "args": {
      "External id": 2482176,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9879
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660281.084, "dur": 23.209,
    "args": {
      "External id": 2482177,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9880
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660305.944, "dur": 2.959,
    "args": {
      "External id": 2482178,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9881
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660307.936, "dur": 0.628,
    "args": {
      "External id": 2482179,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9882
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660309.663, "dur": 20.052,
    "args": {
      "External id": 2482180,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9883
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660310.440, "dur": 1.533,
    "args": {
      "External id": 2482181,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9884
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660313.044, "dur": 16.375,
    "args": {
      "External id": 2482182,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9885
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660316.943, "dur": 2.253,
    "args": {
      "External id": 2482183,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9886
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660333.061, "dur": 19.374,
    "args": {
      "External id": 2482184,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9887
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660353.618, "dur": 11.684,
    "args": {
      "External id": 2482185,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9888
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660367.730, "dur": 12.802,
    "args": {
      "External id": 2482186,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9889
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660381.706, "dur": 11.492,
    "args": {
      "External id": 2482187,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9890
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660394.906, "dur": 17.061,
    "args": {
      "External id": 2482188,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9891
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660396.208, "dur": 1.120,
    "args": {
      "External id": 2482189,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9892
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660399.224, "dur": 0.528,
    "args": {
      "External id": 2482190,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9893
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660413.605, "dur": 11.443,
    "args": {
      "External id": 2482191,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9894
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660427.734, "dur": 9.403,
    "args": {
      "External id": 2482192,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9895
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660444.439, "dur": 1.455,
    "args": {
      "External id": 2482193,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9896
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660454.189, "dur": 3.375,
    "args": {
      "External id": 2482194,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9897
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660456.424, "dur": 0.313,
    "args": {
      "External id": 2482195,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "49152"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9898
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660515.770, "dur": 70.267,
    "args": {
      "External id": 2482196,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9899
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660592.865, "dur": 5.715,
    "args": {
      "External id": 2482197,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9900
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660595.879, "dur": 1.240,
    "args": {
      "External id": 2482198,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "100663296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9901
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660600.089, "dur": 25.312,
    "args": {
      "External id": 2482199,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9902
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660632.007, "dur": 5.182,
    "args": {
      "External id": 2482200,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9903
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660633.719, "dur": 2.758,
    "args": {
      "External id": 2482201,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9904
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660635.385, "dur": 0.897,
    "args": {
      "External id": 2482202,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9905
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660640.245, "dur": 41.927,
    "args": {
      "External id": 2482203,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9906
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660641.167, "dur": 40.358,
    "args": {
      "External id": 2482204,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9907
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660685.878, "dur": 14.266,
    "args": {
      "External id": 2482205,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9908
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660705.869, "dur": 5.193,
    "args": {
      "External id": 2482206,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9909
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660709.552, "dur": 0.653,
    "args": {
      "External id": 2482207,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "117440512"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9910
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660714.666, "dur": 50.196,
    "args": {
      "External id": 2482208,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9911
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660715.342, "dur": 5.602,
    "args": {
      "External id": 2482209,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9912
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660715.934, "dur": 4.445,
    "args": {
      "External id": 2482210,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9913
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660719.403, "dur": 0.800,
    "args": {
      "External id": 2482211,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9914
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660721.511, "dur": 42.971,
    "args": {
      "External id": 2482212,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9915
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660723.976, "dur": 40.042,
    "args": {
      "External id": 2482213,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9916
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660768.463, "dur": 3.784,
    "args": {
      "External id": 2482214,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9917
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660770.348, "dur": 0.753,
    "args": {
      "External id": 2482215,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "57344"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9918
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660777.961, "dur": 1.514,
    "args": {
      "External id": 2482216,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9919
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660786.291, "dur": 7.642,
    "args": {
      "External id": 2482217,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9920
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660789.424, "dur": 4.248,
    "args": {
      "External id": 2482218,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9921
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660868.909, "dur": 207.063,
    "args": {
      "External id": 2482219,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9922
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660871.150, "dur": 2.051,
    "args": {
      "External id": 2482220,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9923
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660874.309, "dur": 201.169,
    "args": {
      "External id": 2482221,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9924
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660877.255, "dur": 0.286,
    "args": {
      "External id": 2482222,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9925
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660878.665, "dur": 19.205,
    "args": {
      "External id": 2482223,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9926
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660899.450, "dur": 3.153,
    "args": {
      "External id": 2482224,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9927
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660901.495, "dur": 0.850,
    "args": {
      "External id": 2482225,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9928
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660903.286, "dur": 26.399,
    "args": {
      "External id": 2482226,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9929
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660911.691, "dur": 1.354,
    "args": {
      "External id": 2482227,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9930
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660913.923, "dur": 15.423,
    "args": {
      "External id": 2482228,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9931
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660916.194, "dur": 2.359,
    "args": {
      "External id": 2482229,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9932
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660930.876, "dur": 17.648,
    "args": {
      "External id": 2482230,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9933
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660949.572, "dur": 11.724,
    "args": {
      "External id": 2482231,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9934
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660963.578, "dur": 12.795,
    "args": {
      "External id": 2482232,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9935
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656660977.663, "dur": 45.904,
    "args": {
      "External id": 2482233,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9936
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661026.811, "dur": 22.093,
    "args": {
      "External id": 2482234,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9937
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661028.562, "dur": 1.930,
    "args": {
      "External id": 2482235,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9938
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661034.184, "dur": 0.796,
    "args": {
      "External id": 2482236,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9939
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661050.642, "dur": 11.651,
    "args": {
      "External id": 2482237,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9940
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661063.444, "dur": 11.072,
    "args": {
      "External id": 2482238,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9941
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661082.973, "dur": 2.149,
    "args": {
      "External id": 2482239,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9942
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661093.163, "dur": 3.523,
    "args": {
      "External id": 2482240,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9943
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661095.484, "dur": 0.357,
    "args": {
      "External id": 2482241,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "57344"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9944
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661158.687, "dur": 53.115,
    "args": {
      "External id": 2482242,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9945
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661216.140, "dur": 8.386,
    "args": {
      "External id": 2482243,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9946
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661220.747, "dur": 2.591,
    "args": {
      "External id": 2482244,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "117440512"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9947
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661225.840, "dur": 25.337,
    "args": {
      "External id": 2482245,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9948
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661255.458, "dur": 4.926,
    "args": {
      "External id": 2482246,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9949
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661256.792, "dur": 2.891,
    "args": {
      "External id": 2482247,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9950
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661258.430, "dur": 0.993,
    "args": {
      "External id": 2482248,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9951
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661262.457, "dur": 42.752,
    "args": {
      "External id": 2482249,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9952
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661265.296, "dur": 39.482,
    "args": {
      "External id": 2482250,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9953
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661308.805, "dur": 13.826,
    "args": {
      "External id": 2482251,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9954
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661327.698, "dur": 22.141,
    "args": {
      "External id": 2482252,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", ""], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 9955
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661329.903, "dur": 19.553,
    "args": {
      "External id": 2482253,"Record function id": 0, "Concrete Inputs": ["", "[]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9956
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661335.186, "dur": 1.275,
    "args": {
      "External id": 2482254,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 9957
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661354.713, "dur": 26.207,
    "args": {
      "External id": 2482255,"Record function id": 0, "Concrete Inputs": ["", "", "15", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], []], "Ev Idx": 9958
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661356.485, "dur": 24.229,
    "args": {
      "External id": 2482256,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], [], []], "Ev Idx": 9959
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661360.725, "dur": 4.034,
    "args": {
      "External id": 2482257,"Record function id": 0, "Concrete Inputs": ["[32000, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9960
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661367.561, "dur": 12.711,
    "args": {
      "External id": 2482258,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "float", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9961
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661392.409, "dur": 4.906,
    "args": {
      "External id": 2482259,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 9962
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661394.260, "dur": 2.648,
    "args": {
      "External id": 2482260,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 9963
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661398.503, "dur": 1.171,
    "args": {
      "External id": 2482261,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9964
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661399.159, "dur": 0.438,
    "args": {
      "External id": 2482262,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9965
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661440.268, "dur": 19.166,
    "args": {
      "External id": 2482263,"Sequence number": 24740640, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[1], [], []], "Ev Idx": 9966
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661461.606, "dur": 13.945,
    "args": {
      "External id": 2482264,"Sequence number": 24740641, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[1], [], []], "Ev Idx": 9967
    }
  },
  {
    "ph": "s", "id": 9, "pid": 1336754, "tid": 1336754, "ts": 1514656661461.606,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661480.543, "dur": 5.489,
    "args": {
      "External id": 2482265,"Sequence number": 24740642, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], [], []], "Input Dims": [[16, 4, 4096], [], [], [], []], "Ev Idx": 9968
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661483.662, "dur": 1.063,
    "args": {
      "External id": 2482266,"Record function id": 0, "Concrete Inputs": ["", "[16, 4, 4096]", "[8192, 1, 1]", "1"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 4, 4096], [], [], []], "Ev Idx": 9969
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661493.908, "dur": 7.512,
    "args": {
      "External id": 2482267,"Sequence number": 24740642, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "3"], "Input type": ["long int", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], []], "Input Dims": [[16, 4, 4096], [], []], "Ev Idx": 9970
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661499.616, "dur": 0.649,
    "args": {
      "External id": 2482268,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096]", "[8192, 1]", "4"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 4, 4096], [], [], []], "Ev Idx": 9971
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661502.668, "dur": 2.511,
    "args": {
      "External id": 2482269,"Sequence number": 24740642, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "0", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], []], "Ev Idx": 9972
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661504.102, "dur": 0.501,
    "args": {
      "External id": 2482270,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096]", "[8192, 1]", "4"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1], [], [], []], "Input Dims": [[16, 4096], [], [], []], "Ev Idx": 9973
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661510.935, "dur": 5.636,
    "args": {
      "External id": 2482271,"Sequence number": 24740642, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 9974
    }
  },
  {
    "ph": "s", "id": 8, "pid": 1336754, "tid": 1336754, "ts": 1514656661510.935,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661514.279, "dur": 1.012,
    "args": {
      "External id": 2482272,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 9975
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661517.570, "dur": 3.827,
    "args": {
      "External id": 2482273,"Sequence number": 24740643, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 9976
    }
  },
  {
    "ph": "s", "id": 7, "pid": 1336754, "tid": 1336754, "ts": 1514656661517.570,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661520.223, "dur": 0.417,
    "args": {
      "External id": 2482274,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 9977
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661522.304, "dur": 23.658,
    "args": {
      "External id": 2482275,"Sequence number": 24740644, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "2", "3"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], []], "Input Dims": [[16, 4096, 4, 2048], [], []], "Ev Idx": 9978
    }
  },
  {
    "ph": "s", "id": 6, "pid": 1336754, "tid": 1336754, "ts": 1514656661522.304,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661543.107, "dur": 0.970,
    "args": {
      "External id": 2482276,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "6144"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 9979
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661547.832, "dur": 6.197,
    "args": {
      "External id": 2482277,"Sequence number": 24740645, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "2", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 1], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], []], "Ev Idx": 9980
    }
  },
  {
    "ph": "s", "id": 5, "pid": 1336754, "tid": 1336754, "ts": 1514656661547.832,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661551.002, "dur": 2.087,
    "args": {
      "External id": 2482278,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "6144"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 1], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], []], "Ev Idx": 9981
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::contiguous", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661558.688, "dur": 33.808,
    "args": {
      "External id": 2482279,"Sequence number": 24740646, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["long int", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 9982
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661560.042, "dur": 32.257,
    "args": {
      "External id": 2482280,"Sequence number": 24740646, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["long int", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 9983
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661562.561, "dur": 6.549,
    "args": {
      "External id": 2482281,"Record function id": 0, "Concrete Inputs": ["", "4", "0", "", "", "0"], "Input type": ["long int", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[8192, 1], [], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], [], []], "Ev Idx": 9984
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661564.564, "dur": 3.908,
    "args": {
      "External id": 2482282,"Record function id": 0, "Concrete Inputs": ["[16, 4096]", "4", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9985
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661570.429, "dur": 21.449,
    "args": {
      "External id": 2482283,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[4096, 1], [8192, 1], []], "Input Dims": [[16, 4096], [16, 4096], []], "Ev Idx": 9986
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661618.710, "dur": 5.022,
    "args": {
      "External id": 2482284,"Sequence number": 24740646, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 9987
    }
  },
  {
    "ph": "s", "id": 4, "pid": 1336754, "tid": 1336754, "ts": 1514656661618.710,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661626.258, "dur": 1.250,
    "args": {
      "External id": 2482285,"Sequence number": 24740647, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["long int", "ScalarList"], "Input Strides": [[4096, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 9988
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FusedLinearCrossEntropyFunction", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661659.043, "dur": 44102.473,
    "args": {
      "External id": 2482286,"Sequence number": 24740647, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "-100", "0.", "1.", "8"], "Input type": ["c10::BFloat16", "long int", "c10::BFloat16", "", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1], [1], [2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536], [32000, 2048], [], [], [], [], []], "Ev Idx": 9989
    }
  },
  {
    "ph": "s", "id": 3, "pid": 1336754, "tid": 1336754, "ts": 1514656661659.043,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::contiguous", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661673.843, "dur": 26.795,
    "args": {
      "External id": 2482287,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 9990
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661674.376, "dur": 26.048,
    "args": {
      "External id": 2482288,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 9991
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661675.543, "dur": 7.425,
    "args": {
      "External id": 2482289,"Record function id": 0, "Concrete Inputs": ["", "15", "0", "", "", "0"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[8192, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 9992
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661677.123, "dur": 5.128,
    "args": {
      "External id": 2482290,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9993
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661683.667, "dur": 16.429,
    "args": {
      "External id": 2482291,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [8192, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 9994
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661715.876, "dur": 26.105,
    "args": {
      "External id": 2482292,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 9995
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661717.103, "dur": 8.692,
    "args": {
      "External id": 2482293,"Record function id": 0, "Concrete Inputs": ["", "15", "0", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 9996
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661720.990, "dur": 4.272,
    "args": {
      "External id": 2482294,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9997
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661726.999, "dur": 14.759,
    "args": {
      "External id": 2482295,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 9998
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661728.640, "dur": 12.640,
    "args": {
      "External id": 2482296,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 9999
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661745.362, "dur": 17.614,
    "args": {
      "External id": 2482297,"Record function id": 0, "Concrete Inputs": ["", "6", "", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], []], "Ev Idx": 10000
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661745.849, "dur": 4.169,
    "args": {
      "External id": 2482298,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], []], "Ev Idx": 10001
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661747.162, "dur": 2.587,
    "args": {
      "External id": 2482299,"Record function id": 0, "Concrete Inputs": ["[32000, 2048]", "[2048, 1]", "6", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10002
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661750.464, "dur": 12.321,
    "args": {
      "External id": 2482300,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 10003
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661751.208, "dur": 11.274,
    "args": {
      "External id": 2482301,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[2048, 1], []], "Input Dims": [[32000, 2048], []], "Ev Idx": 10004
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661768.299, "dur": 17.784,
    "args": {
      "External id": 2482302,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False"], "Input type": ["ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 10005
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661771.274, "dur": 3.508,
    "args": {
      "External id": 2482303,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10006
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661775.315, "dur": 10.511,
    "args": {
      "External id": 2482304,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[65536]], "Ev Idx": 10007
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661776.142, "dur": 9.397,
    "args": {
      "External id": 2482305,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 10008
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::ne", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661790.390, "dur": 20.052,
    "args": {
      "External id": 2482306,"Record function id": 0, "Concrete Inputs": ["", "-100"], "Input type": ["long int", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 10009
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661813.123, "dur": 44.605,
    "args": {
      "External id": 2482307,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["bool", ""], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 10010
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661814.629, "dur": 42.637,
    "args": {
      "External id": 2482308,"Record function id": 0, "Concrete Inputs": ["", "[]", "False", ""], "Input type": ["bool", "ScalarList", "Scalar", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10011
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661820.440, "dur": 1.126,
    "args": {
      "External id": 2482309,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[0]", ""], "Input type": ["long int", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10012
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661822.867, "dur": 20.401,
    "args": {
      "External id": 2482310,"Record function id": 0, "Concrete Inputs": ["", "4", "False", "False", ""], "Input type": ["bool", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10013
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661824.369, "dur": 18.574,
    "args": {
      "External id": 2482311,"Record function id": 0, "Concrete Inputs": ["", "4", "", "", "", "False", ""], "Input type": ["bool", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[65536], [], [], [], [], [], []], "Ev Idx": 10014
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661826.435, "dur": 2.756,
    "args": {
      "External id": 2482312,"Record function id": 0, "Concrete Inputs": ["[65536]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10015
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661830.086, "dur": 12.451,
    "args": {
      "External id": 2482313,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["long int", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[65536], [65536], []], "Ev Idx": 10016
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::item", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661863.332, "dur": 38382.854,
    "args": {
      "External id": 2482314,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["long int"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 10017
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_local_scalar_dense", "pid": 1336754, "tid": 1336754,
    "ts": 1514656661865.120, "dur": 38379.899,
    "args": {
      "External id": 2482315,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["long int"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 10018
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656700257.696, "dur": 6.821,
    "args": {
      "External id": 2482316,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10019
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656700261.400, "dur": 1.452,
    "args": {
      "External id": 2482317,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10020
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656700269.780, "dur": 104.239,
    "args": {
      "External id": 2482318,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 10021
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656700271.236, "dur": 7.221,
    "args": {
      "External id": 2482319,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 10022
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656700273.109, "dur": 4.645,
    "args": {
      "External id": 2482320,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 10023
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656700276.612, "dur": 0.846,
    "args": {
      "External id": 2482321,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 10024
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656700279.908, "dur": 93.476,
    "args": {
      "External id": 2482322,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10025
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656700281.718, "dur": 90.844,
    "args": {
      "External id": 2482323,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10026
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656700377.850, "dur": 4.612,
    "args": {
      "External id": 2482324,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10027
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656700380.167, "dur": 1.095,
    "args": {
      "External id": 2482325,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "0"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10028
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656700389.817, "dur": 2.392,
    "args": {
      "External id": 2482326,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 10029
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656700401.876, "dur": 6.939,
    "args": {
      "External id": 2482327,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 10030
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656700404.179, "dur": 4.363,
    "args": {
      "External id": 2482328,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10031
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656700548.087, "dur": 198.122,
    "args": {
      "External id": 2482329,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10032
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656700552.387, "dur": 3.205,
    "args": {
      "External id": 2482330,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10033
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656700557.183, "dur": 188.541,
    "args": {
      "External id": 2482331,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 10034
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336754, "tid": 1336754,
    "ts": 1514656700558.681, "dur": 0.597,
    "args": {
      "External id": 2482332,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10035
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336754, "tid": 1336754,
    "ts": 1514656700560.628, "dur": 26.851,
    "args": {
      "External id": 2482333,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10036
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514656700590.938, "dur": 5.268,
    "args": {
      "External id": 2482334,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 10037
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656700594.706, "dur": 1.127,
    "args": {
      "External id": 2482335,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 10038
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656700597.338, "dur": 22.908,
    "args": {
      "External id": 2482336,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10039
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656700598.292, "dur": 1.309,
    "args": {
      "External id": 2482337,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10040
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656700600.852, "dur": 19.136,
    "args": {
      "External id": 2482338,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 10041
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656700604.234, "dur": 3.202,
    "args": {
      "External id": 2482339,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10042
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336754, "tid": 1336754,
    "ts": 1514656700621.803, "dur": 21.556,
    "args": {
      "External id": 2482340,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10043
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656700645.120, "dur": 17.311,
    "args": {
      "External id": 2482341,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10044
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336754, "tid": 1336754,
    "ts": 1514656700665.290, "dur": 14.292,
    "args": {
      "External id": 2482342,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 10045
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656700681.082, "dur": 12.832,
    "args": {
      "External id": 2482343,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10046
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1336754,
    "ts": 1514656700697.621, "dur": 19.938,
    "args": {
      "External id": 2482344,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 10047
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656700699.760, "dur": 1.261,
    "args": {
      "External id": 2482345,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10048
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656700703.015, "dur": 0.946,
    "args": {
      "External id": 2482346,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 10049
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656700719.375, "dur": 12.327,
    "args": {
      "External id": 2482347,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10050
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656700732.968, "dur": 11.378,
    "args": {
      "External id": 2482348,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10051
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656700753.852, "dur": 2.193,
    "args": {
      "External id": 2482349,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10052
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656700765.250, "dur": 4.201,
    "args": {
      "External id": 2482350,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10053
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656700767.588, "dur": 0.808,
    "args": {
      "External id": 2482351,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10054
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656700846.325, "dur": 69.191,
    "args": {
      "External id": 2482352,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 10055
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656700920.768, "dur": 5.379,
    "args": {
      "External id": 2482353,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10056
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656700923.167, "dur": 0.803,
    "args": {
      "External id": 2482354,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10057
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656700927.558, "dur": 25.786,
    "args": {
      "External id": 2482355,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 10058
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656700958.652, "dur": 7.558,
    "args": {
      "External id": 2482356,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 10059
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656700960.249, "dur": 5.316,
    "args": {
      "External id": 2482357,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 10060
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656700963.877, "dur": 1.411,
    "args": {
      "External id": 2482358,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 10061
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656700968.944, "dur": 85.142,
    "args": {
      "External id": 2482359,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10062
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656700970.211, "dur": 82.635,
    "args": {
      "External id": 2482360,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10063
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701060.115, "dur": 16.754,
    "args": {
      "External id": 2482361,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 10064
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701083.916, "dur": 5.050,
    "args": {
      "External id": 2482362,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10065
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701086.652, "dur": 1.129,
    "args": {
      "External id": 2482363,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "16777216"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10066
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701095.283, "dur": 50.685,
    "args": {
      "External id": 2482364,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 10067
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701096.371, "dur": 3.866,
    "args": {
      "External id": 2482365,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 10068
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701097.365, "dur": 2.257,
    "args": {
      "External id": 2482366,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 10069
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701098.877, "dur": 0.599,
    "args": {
      "External id": 2482367,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 10070
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701100.837, "dur": 44.778,
    "args": {
      "External id": 2482368,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10071
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701101.843, "dur": 43.303,
    "args": {
      "External id": 2482369,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10072
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701151.774, "dur": 4.031,
    "args": {
      "External id": 2482370,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10073
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701153.906, "dur": 0.801,
    "args": {
      "External id": 2482371,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "8192"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10074
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701162.298, "dur": 1.602,
    "args": {
      "External id": 2482372,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 10075
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701171.936, "dur": 7.769,
    "args": {
      "External id": 2482373,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 10076
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701173.880, "dur": 5.538,
    "args": {
      "External id": 2482374,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10077
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701277.547, "dur": 173.892,
    "args": {
      "External id": 2482375,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10078
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701279.602, "dur": 2.926,
    "args": {
      "External id": 2482376,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10079
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701283.767, "dur": 167.281,
    "args": {
      "External id": 2482377,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 10080
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701285.183, "dur": 0.232,
    "args": {
      "External id": 2482378,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10081
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701286.385, "dur": 22.015,
    "args": {
      "External id": 2482379,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10082
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701310.013, "dur": 3.285,
    "args": {
      "External id": 2482380,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 10083
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701312.038, "dur": 0.892,
    "args": {
      "External id": 2482381,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 10084
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701313.982, "dur": 24.088,
    "args": {
      "External id": 2482382,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10085
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701315.225, "dur": 1.722,
    "args": {
      "External id": 2482383,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10086
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701318.062, "dur": 19.696,
    "args": {
      "External id": 2482384,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 10087
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701323.905, "dur": 2.877,
    "args": {
      "External id": 2482385,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10088
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701339.420, "dur": 20.812,
    "args": {
      "External id": 2482386,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10089
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701361.812, "dur": 12.541,
    "args": {
      "External id": 2482387,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10090
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701377.001, "dur": 13.060,
    "args": {
      "External id": 2482388,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 10091
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701391.237, "dur": 11.503,
    "args": {
      "External id": 2482389,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10092
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701404.391, "dur": 19.097,
    "args": {
      "External id": 2482390,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 10093
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701406.264, "dur": 1.287,
    "args": {
      "External id": 2482391,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10094
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701409.413, "dur": 1.174,
    "args": {
      "External id": 2482392,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 10095
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701426.972, "dur": 11.531,
    "args": {
      "External id": 2482393,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10096
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701439.500, "dur": 10.771,
    "args": {
      "External id": 2482394,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10097
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701457.931, "dur": 1.904,
    "args": {
      "External id": 2482395,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10098
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701467.648, "dur": 3.660,
    "args": {
      "External id": 2482396,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10099
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701469.917, "dur": 0.582,
    "args": {
      "External id": 2482397,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "8192"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10100
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701551.909, "dur": 57.503,
    "args": {
      "External id": 2482398,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 10101
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701615.281, "dur": 5.937,
    "args": {
      "External id": 2482399,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10102
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701618.346, "dur": 1.340,
    "args": {
      "External id": 2482400,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "16777216"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10103
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701624.809, "dur": 24.339,
    "args": {
      "External id": 2482401,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 10104
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701654.050, "dur": 4.848,
    "args": {
      "External id": 2482402,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 10105
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701655.348, "dur": 2.943,
    "args": {
      "External id": 2482403,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 10106
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701657.050, "dur": 1.057,
    "args": {
      "External id": 2482404,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 10107
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701661.125, "dur": 43.678,
    "args": {
      "External id": 2482405,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10108
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701662.121, "dur": 42.170,
    "args": {
      "External id": 2482406,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10109
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701708.622, "dur": 14.634,
    "args": {
      "External id": 2482407,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 10110
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701731.111, "dur": 3.484,
    "args": {
      "External id": 2482408,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10111
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701733.140, "dur": 0.674,
    "args": {
      "External id": 2482409,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "33554432"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10112
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701738.525, "dur": 50.776,
    "args": {
      "External id": 2482410,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 10113
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701739.342, "dur": 5.392,
    "args": {
      "External id": 2482411,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 10114
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701739.970, "dur": 4.138,
    "args": {
      "External id": 2482412,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 10115
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701741.189, "dur": 2.695,
    "args": {
      "External id": 2482413,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 10116
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701746.908, "dur": 42.028,
    "args": {
      "External id": 2482414,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10117
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701747.672, "dur": 40.562,
    "args": {
      "External id": 2482415,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10118
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701793.569, "dur": 3.567,
    "args": {
      "External id": 2482416,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10119
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701795.319, "dur": 0.782,
    "args": {
      "External id": 2482417,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "16384"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10120
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701803.041, "dur": 1.528,
    "args": {
      "External id": 2482418,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 10121
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701811.479, "dur": 5.948,
    "args": {
      "External id": 2482419,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 10122
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701813.369, "dur": 3.790,
    "args": {
      "External id": 2482420,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10123
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701903.112, "dur": 210.590,
    "args": {
      "External id": 2482421,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10124
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701905.089, "dur": 2.295,
    "args": {
      "External id": 2482422,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10125
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701908.646, "dur": 204.605,
    "args": {
      "External id": 2482423,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 10126
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701909.816, "dur": 0.380,
    "args": {
      "External id": 2482424,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10127
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701912.829, "dur": 19.087,
    "args": {
      "External id": 2482425,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10128
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701933.619, "dur": 2.828,
    "args": {
      "External id": 2482426,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 10129
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701935.569, "dur": 0.651,
    "args": {
      "External id": 2482427,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 10130
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701937.074, "dur": 22.965,
    "args": {
      "External id": 2482428,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10131
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701938.306, "dur": 3.007,
    "args": {
      "External id": 2482429,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10132
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701942.394, "dur": 17.278,
    "args": {
      "External id": 2482430,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 10133
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701946.439, "dur": 2.676,
    "args": {
      "External id": 2482431,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10134
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336754, "tid": 1336754,
    "ts": 1514656701961.398, "dur": 54.887,
    "args": {
      "External id": 2482432,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10135
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702019.239, "dur": 14.652,
    "args": {
      "External id": 2482433,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10136
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702037.173, "dur": 13.596,
    "args": {
      "External id": 2482434,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 10137
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702052.082, "dur": 11.441,
    "args": {
      "External id": 2482435,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10138
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702065.180, "dur": 19.935,
    "args": {
      "External id": 2482436,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 10139
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702067.414, "dur": 1.787,
    "args": {
      "External id": 2482437,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10140
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702071.091, "dur": 0.901,
    "args": {
      "External id": 2482438,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 10141
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702088.326, "dur": 11.795,
    "args": {
      "External id": 2482439,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10142
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702101.170, "dur": 10.917,
    "args": {
      "External id": 2482440,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10143
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702121.432, "dur": 2.503,
    "args": {
      "External id": 2482441,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10144
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702132.524, "dur": 3.700,
    "args": {
      "External id": 2482442,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10145
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702134.767, "dur": 0.592,
    "args": {
      "External id": 2482443,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "16384"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10146
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702201.786, "dur": 73.678,
    "args": {
      "External id": 2482444,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 10147
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702280.210, "dur": 7.048,
    "args": {
      "External id": 2482445,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10148
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702284.985, "dur": 1.238,
    "args": {
      "External id": 2482446,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "33554432"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10149
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702288.417, "dur": 29.351,
    "args": {
      "External id": 2482447,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 10150
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702322.518, "dur": 7.122,
    "args": {
      "External id": 2482448,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 10151
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702324.068, "dur": 4.878,
    "args": {
      "External id": 2482449,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 10152
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702325.932, "dur": 2.793,
    "args": {
      "External id": 2482450,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 10153
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702331.987, "dur": 74.110,
    "args": {
      "External id": 2482451,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10154
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702332.914, "dur": 72.628,
    "args": {
      "External id": 2482452,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10155
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702411.527, "dur": 33.798,
    "args": {
      "External id": 2482453,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 10156
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702450.797, "dur": 3.816,
    "args": {
      "External id": 2482454,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10157
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702453.044, "dur": 0.768,
    "args": {
      "External id": 2482455,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "50331648"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10158
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702458.345, "dur": 52.240,
    "args": {
      "External id": 2482456,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 10159
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702459.369, "dur": 5.155,
    "args": {
      "External id": 2482457,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 10160
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702459.887, "dur": 4.128,
    "args": {
      "External id": 2482458,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 10161
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702462.961, "dur": 0.703,
    "args": {
      "External id": 2482459,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 10162
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702465.168, "dur": 45.089,
    "args": {
      "External id": 2482460,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10163
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702466.140, "dur": 43.625,
    "args": {
      "External id": 2482461,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10164
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702514.477, "dur": 30.778,
    "args": {
      "External id": 2482462,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10165
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702542.061, "dur": 1.201,
    "args": {
      "External id": 2482463,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "24576"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10166
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702553.041, "dur": 1.664,
    "args": {
      "External id": 2482464,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 10167
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702564.564, "dur": 7.686,
    "args": {
      "External id": 2482465,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 10168
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702566.246, "dur": 5.721,
    "args": {
      "External id": 2482466,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10169
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702663.756, "dur": 173.434,
    "args": {
      "External id": 2482467,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10170
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702665.873, "dur": 2.507,
    "args": {
      "External id": 2482468,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10171
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702669.764, "dur": 166.842,
    "args": {
      "External id": 2482469,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 10172
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702671.304, "dur": 0.292,
    "args": {
      "External id": 2482470,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10173
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702672.547, "dur": 23.284,
    "args": {
      "External id": 2482471,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10174
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702699.096, "dur": 3.000,
    "args": {
      "External id": 2482472,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 10175
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702701.012, "dur": 0.785,
    "args": {
      "External id": 2482473,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 10176
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702702.795, "dur": 22.322,
    "args": {
      "External id": 2482474,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10177
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702704.046, "dur": 1.730,
    "args": {
      "External id": 2482475,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10178
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702706.888, "dur": 17.950,
    "args": {
      "External id": 2482476,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 10179
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702710.916, "dur": 2.438,
    "args": {
      "External id": 2482477,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10180
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702726.535, "dur": 19.772,
    "args": {
      "External id": 2482478,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10181
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702747.710, "dur": 12.130,
    "args": {
      "External id": 2482479,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10182
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702762.702, "dur": 14.063,
    "args": {
      "External id": 2482480,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 10183
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702778.226, "dur": 11.560,
    "args": {
      "External id": 2482481,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10184
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702792.944, "dur": 18.056,
    "args": {
      "External id": 2482482,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 10185
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702795.204, "dur": 1.412,
    "args": {
      "External id": 2482483,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10186
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702798.295, "dur": 0.806,
    "args": {
      "External id": 2482484,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 10187
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702812.483, "dur": 11.363,
    "args": {
      "External id": 2482485,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10188
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702824.912, "dur": 10.934,
    "args": {
      "External id": 2482486,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10189
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702843.257, "dur": 1.725,
    "args": {
      "External id": 2482487,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10190
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702853.220, "dur": 3.452,
    "args": {
      "External id": 2482488,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10191
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702855.492, "dur": 0.395,
    "args": {
      "External id": 2482489,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "24576"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10192
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656702920.766, "dur": 57.617,
    "args": {
      "External id": 2482490,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 10193
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703020.801, "dur": 6.787,
    "args": {
      "External id": 2482491,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10194
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703024.194, "dur": 1.753,
    "args": {
      "External id": 2482492,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "50331648"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10195
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703028.813, "dur": 27.878,
    "args": {
      "External id": 2482493,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 10196
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703062.803, "dur": 7.536,
    "args": {
      "External id": 2482494,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 10197
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703064.598, "dur": 5.060,
    "args": {
      "External id": 2482495,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 10198
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703068.146, "dur": 1.310,
    "args": {
      "External id": 2482496,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 10199
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703073.341, "dur": 48.219,
    "args": {
      "External id": 2482497,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10200
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703074.327, "dur": 46.347,
    "args": {
      "External id": 2482498,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10201
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703125.889, "dur": 15.220,
    "args": {
      "External id": 2482499,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 10202
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703146.746, "dur": 3.806,
    "args": {
      "External id": 2482500,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10203
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703148.742, "dur": 0.899,
    "args": {
      "External id": 2482501,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "67108864"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10204
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703154.799, "dur": 51.540,
    "args": {
      "External id": 2482502,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 10205
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703157.377, "dur": 5.360,
    "args": {
      "External id": 2482503,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 10206
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703158.358, "dur": 3.730,
    "args": {
      "External id": 2482504,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 10207
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703159.597, "dur": 2.187,
    "args": {
      "External id": 2482505,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 10208
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703163.284, "dur": 42.669,
    "args": {
      "External id": 2482506,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10209
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703164.020, "dur": 41.410,
    "args": {
      "External id": 2482507,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10210
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703210.271, "dur": 3.988,
    "args": {
      "External id": 2482508,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10211
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703212.449, "dur": 0.791,
    "args": {
      "External id": 2482509,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "32768"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10212
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703220.064, "dur": 1.417,
    "args": {
      "External id": 2482510,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 10213
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703230.284, "dur": 5.613,
    "args": {
      "External id": 2482511,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 10214
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703231.920, "dur": 3.726,
    "args": {
      "External id": 2482512,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10215
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703322.847, "dur": 175.646,
    "args": {
      "External id": 2482513,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10216
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703325.318, "dur": 2.092,
    "args": {
      "External id": 2482514,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10217
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703328.992, "dur": 169.086,
    "args": {
      "External id": 2482515,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 10218
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703329.989, "dur": 0.541,
    "args": {
      "External id": 2482516,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10219
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703334.960, "dur": 19.883,
    "args": {
      "External id": 2482517,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10220
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703360.743, "dur": 4.563,
    "args": {
      "External id": 2482518,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 10221
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703364.031, "dur": 0.827,
    "args": {
      "External id": 2482519,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 10222
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703366.057, "dur": 21.819,
    "args": {
      "External id": 2482520,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10223
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703367.562, "dur": 2.841,
    "args": {
      "External id": 2482521,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10224
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703371.607, "dur": 15.853,
    "args": {
      "External id": 2482522,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 10225
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703374.077, "dur": 2.473,
    "args": {
      "External id": 2482523,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10226
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703389.296, "dur": 18.526,
    "args": {
      "External id": 2482524,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10227
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703409.346, "dur": 12.688,
    "args": {
      "External id": 2482525,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10228
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703426.859, "dur": 12.863,
    "args": {
      "External id": 2482526,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 10229
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703440.849, "dur": 12.174,
    "args": {
      "External id": 2482527,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10230
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703454.695, "dur": 17.848,
    "args": {
      "External id": 2482528,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 10231
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703456.814, "dur": 1.081,
    "args": {
      "External id": 2482529,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10232
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703459.789, "dur": 0.817,
    "args": {
      "External id": 2482530,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 10233
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703474.072, "dur": 11.257,
    "args": {
      "External id": 2482531,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10234
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703486.463, "dur": 10.848,
    "args": {
      "External id": 2482532,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10235
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703506.385, "dur": 1.435,
    "args": {
      "External id": 2482533,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10236
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703515.616, "dur": 3.161,
    "args": {
      "External id": 2482534,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10237
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703517.653, "dur": 0.320,
    "args": {
      "External id": 2482535,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "32768"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10238
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703601.101, "dur": 55.430,
    "args": {
      "External id": 2482536,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 10239
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703661.421, "dur": 5.795,
    "args": {
      "External id": 2482537,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10240
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703664.248, "dur": 1.408,
    "args": {
      "External id": 2482538,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "67108864"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10241
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703668.417, "dur": 23.002,
    "args": {
      "External id": 2482539,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 10242
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703696.101, "dur": 8.923,
    "args": {
      "External id": 2482540,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 10243
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703699.641, "dur": 4.773,
    "args": {
      "External id": 2482541,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 10244
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703701.141, "dur": 3.066,
    "args": {
      "External id": 2482542,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 10245
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703707.716, "dur": 39.782,
    "args": {
      "External id": 2482543,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10246
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703708.860, "dur": 37.924,
    "args": {
      "External id": 2482544,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10247
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703751.321, "dur": 14.398,
    "args": {
      "External id": 2482545,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 10248
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703771.344, "dur": 5.305,
    "args": {
      "External id": 2482546,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10249
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703775.111, "dur": 0.762,
    "args": {
      "External id": 2482547,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "83886080"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10250
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703780.194, "dur": 45.982,
    "args": {
      "External id": 2482548,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 10251
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703781.159, "dur": 3.512,
    "args": {
      "External id": 2482549,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 10252
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703782.129, "dur": 2.054,
    "args": {
      "External id": 2482550,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 10253
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703783.304, "dur": 0.633,
    "args": {
      "External id": 2482551,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 10254
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703785.281, "dur": 40.495,
    "args": {
      "External id": 2482552,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10255
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703785.747, "dur": 39.597,
    "args": {
      "External id": 2482553,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10256
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703831.714, "dur": 3.707,
    "args": {
      "External id": 2482554,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10257
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703833.821, "dur": 0.499,
    "args": {
      "External id": 2482555,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "40960"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10258
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703840.695, "dur": 1.533,
    "args": {
      "External id": 2482556,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 10259
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703848.533, "dur": 5.434,
    "args": {
      "External id": 2482557,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 10260
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703849.958, "dur": 3.761,
    "args": {
      "External id": 2482558,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10261
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703931.501, "dur": 213.234,
    "args": {
      "External id": 2482559,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10262
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703933.421, "dur": 2.214,
    "args": {
      "External id": 2482560,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10263
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703939.047, "dur": 205.195,
    "args": {
      "External id": 2482561,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 10264
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703940.045, "dur": 0.227,
    "args": {
      "External id": 2482562,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10265
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703941.236, "dur": 19.167,
    "args": {
      "External id": 2482563,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10266
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703961.923, "dur": 5.238,
    "args": {
      "External id": 2482564,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 10267
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703963.911, "dur": 2.960,
    "args": {
      "External id": 2482565,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 10268
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703968.194, "dur": 61.304,
    "args": {
      "External id": 2482566,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10269
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703969.516, "dur": 1.575,
    "args": {
      "External id": 2482567,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10270
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703972.145, "dur": 56.570,
    "args": {
      "External id": 2482568,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 10271
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656703975.838, "dur": 2.478,
    "args": {
      "External id": 2482569,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10272
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704031.444, "dur": 20.480,
    "args": {
      "External id": 2482570,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10273
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704053.455, "dur": 12.781,
    "args": {
      "External id": 2482571,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10274
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704069.060, "dur": 12.750,
    "args": {
      "External id": 2482572,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 10275
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704082.949, "dur": 11.276,
    "args": {
      "External id": 2482573,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10276
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704096.175, "dur": 20.785,
    "args": {
      "External id": 2482574,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 10277
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704098.134, "dur": 1.930,
    "args": {
      "External id": 2482575,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10278
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704102.057, "dur": 2.344,
    "args": {
      "External id": 2482576,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 10279
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704120.198, "dur": 11.092,
    "args": {
      "External id": 2482577,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10280
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704132.272, "dur": 10.973,
    "args": {
      "External id": 2482578,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10281
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704152.762, "dur": 2.093,
    "args": {
      "External id": 2482579,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10282
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704164.098, "dur": 3.563,
    "args": {
      "External id": 2482580,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10283
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704166.283, "dur": 0.483,
    "args": {
      "External id": 2482581,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "40960"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10284
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704231.061, "dur": 54.511,
    "args": {
      "External id": 2482582,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 10285
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704290.018, "dur": 6.464,
    "args": {
      "External id": 2482583,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10286
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704294.306, "dur": 1.090,
    "args": {
      "External id": 2482584,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "83886080"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10287
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704297.568, "dur": 23.524,
    "args": {
      "External id": 2482585,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 10288
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704325.230, "dur": 5.680,
    "args": {
      "External id": 2482586,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 10289
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704326.916, "dur": 3.314,
    "args": {
      "External id": 2482587,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 10290
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704328.826, "dur": 1.188,
    "args": {
      "External id": 2482588,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 10291
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704333.363, "dur": 40.149,
    "args": {
      "External id": 2482589,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10292
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704334.495, "dur": 38.373,
    "args": {
      "External id": 2482590,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10293
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704378.860, "dur": 14.513,
    "args": {
      "External id": 2482591,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 10294
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704398.727, "dur": 3.504,
    "args": {
      "External id": 2482592,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10295
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704400.535, "dur": 0.775,
    "args": {
      "External id": 2482593,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "100663296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10296
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704405.662, "dur": 48.371,
    "args": {
      "External id": 2482594,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 10297
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704406.592, "dur": 5.013,
    "args": {
      "External id": 2482595,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 10298
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704407.195, "dur": 3.905,
    "args": {
      "External id": 2482596,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 10299
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704410.082, "dur": 0.870,
    "args": {
      "External id": 2482597,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 10300
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704412.112, "dur": 41.529,
    "args": {
      "External id": 2482598,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10301
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704413.029, "dur": 40.061,
    "args": {
      "External id": 2482599,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10302
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704458.016, "dur": 3.749,
    "args": {
      "External id": 2482600,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10303
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704460.050, "dur": 0.702,
    "args": {
      "External id": 2482601,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "49152"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10304
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704466.398, "dur": 1.332,
    "args": {
      "External id": 2482602,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 10305
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704473.953, "dur": 7.582,
    "args": {
      "External id": 2482603,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 10306
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704477.264, "dur": 4.002,
    "args": {
      "External id": 2482604,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10307
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704575.492, "dur": 170.109,
    "args": {
      "External id": 2482605,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10308
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704579.455, "dur": 3.456,
    "args": {
      "External id": 2482606,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10309
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704584.594, "dur": 160.515,
    "args": {
      "External id": 2482607,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 10310
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704586.180, "dur": 0.674,
    "args": {
      "External id": 2482608,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10311
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704587.842, "dur": 20.906,
    "args": {
      "External id": 2482609,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10312
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704610.081, "dur": 4.463,
    "args": {
      "External id": 2482610,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 10313
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704613.195, "dur": 1.036,
    "args": {
      "External id": 2482611,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 10314
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704615.446, "dur": 19.955,
    "args": {
      "External id": 2482612,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10315
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704618.256, "dur": 1.323,
    "args": {
      "External id": 2482613,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10316
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704620.586, "dur": 14.589,
    "args": {
      "External id": 2482614,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 10317
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704622.986, "dur": 2.439,
    "args": {
      "External id": 2482615,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10318
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704636.487, "dur": 19.993,
    "args": {
      "External id": 2482616,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10319
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704657.669, "dur": 11.242,
    "args": {
      "External id": 2482617,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10320
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704671.280, "dur": 13.276,
    "args": {
      "External id": 2482618,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 10321
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704685.874, "dur": 11.060,
    "args": {
      "External id": 2482619,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10322
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704698.396, "dur": 21.342,
    "args": {
      "External id": 2482620,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 10323
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704701.838, "dur": 1.172,
    "args": {
      "External id": 2482621,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10324
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704706.835, "dur": 0.937,
    "args": {
      "External id": 2482622,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 10325
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704721.183, "dur": 11.160,
    "args": {
      "External id": 2482623,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10326
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704733.178, "dur": 11.105,
    "args": {
      "External id": 2482624,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10327
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704751.749, "dur": 1.787,
    "args": {
      "External id": 2482625,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10328
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704761.187, "dur": 3.549,
    "args": {
      "External id": 2482626,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10329
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704763.563, "dur": 0.380,
    "args": {
      "External id": 2482627,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "49152"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10330
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704822.829, "dur": 51.637,
    "args": {
      "External id": 2482628,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 10331
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704878.773, "dur": 6.093,
    "args": {
      "External id": 2482629,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10332
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704882.820, "dur": 1.017,
    "args": {
      "External id": 2482630,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "100663296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10333
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704886.102, "dur": 21.408,
    "args": {
      "External id": 2482631,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 10334
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704911.917, "dur": 5.295,
    "args": {
      "External id": 2482632,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 10335
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704913.451, "dur": 2.947,
    "args": {
      "External id": 2482633,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 10336
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704915.210, "dur": 1.031,
    "args": {
      "External id": 2482634,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 10337
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704919.487, "dur": 40.864,
    "args": {
      "External id": 2482635,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10338
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704922.167, "dur": 37.627,
    "args": {
      "External id": 2482636,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10339
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704963.818, "dur": 13.802,
    "args": {
      "External id": 2482637,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 10340
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656704983.140, "dur": 40.566,
    "args": {
      "External id": 2482638,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10341
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705020.765, "dur": 1.263,
    "args": {
      "External id": 2482639,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "117440512"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10342
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705028.792, "dur": 54.080,
    "args": {
      "External id": 2482640,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 10343
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705029.769, "dur": 5.323,
    "args": {
      "External id": 2482641,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 10344
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705030.861, "dur": 3.725,
    "args": {
      "External id": 2482642,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 10345
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705033.804, "dur": 0.634,
    "args": {
      "External id": 2482643,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 10346
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705035.605, "dur": 46.881,
    "args": {
      "External id": 2482644,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10347
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705036.414, "dur": 45.546,
    "args": {
      "External id": 2482645,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10348
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705087.505, "dur": 3.864,
    "args": {
      "External id": 2482646,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10349
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705089.512, "dur": 0.814,
    "args": {
      "External id": 2482647,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "57344"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10350
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705097.396, "dur": 1.647,
    "args": {
      "External id": 2482648,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 10351
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705107.503, "dur": 5.833,
    "args": {
      "External id": 2482649,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 10352
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705109.216, "dur": 3.853,
    "args": {
      "External id": 2482650,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10353
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705190.216, "dur": 168.913,
    "args": {
      "External id": 2482651,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10354
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705194.022, "dur": 2.025,
    "args": {
      "External id": 2482652,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10355
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705197.212, "dur": 161.562,
    "args": {
      "External id": 2482653,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 10356
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705198.266, "dur": 0.436,
    "args": {
      "External id": 2482654,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10357
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705201.314, "dur": 20.445,
    "args": {
      "External id": 2482655,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10358
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705223.300, "dur": 4.592,
    "args": {
      "External id": 2482656,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 10359
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705226.852, "dur": 0.803,
    "args": {
      "External id": 2482657,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 10360
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705228.724, "dur": 19.288,
    "args": {
      "External id": 2482658,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10361
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705229.686, "dur": 1.513,
    "args": {
      "External id": 2482659,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10362
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705232.315, "dur": 15.440,
    "args": {
      "External id": 2482660,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 10363
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705234.772, "dur": 2.445,
    "args": {
      "External id": 2482661,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10364
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705249.216, "dur": 18.457,
    "args": {
      "External id": 2482662,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10365
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705268.877, "dur": 14.078,
    "args": {
      "External id": 2482663,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10366
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705285.772, "dur": 12.828,
    "args": {
      "External id": 2482664,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 10367
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705301.423, "dur": 11.797,
    "args": {
      "External id": 2482665,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10368
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705314.773, "dur": 17.750,
    "args": {
      "External id": 2482666,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 10369
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705316.501, "dur": 1.506,
    "args": {
      "External id": 2482667,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10370
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705319.714, "dur": 0.811,
    "args": {
      "External id": 2482668,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 10371
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705334.164, "dur": 11.226,
    "args": {
      "External id": 2482669,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10372
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705346.231, "dur": 11.614,
    "args": {
      "External id": 2482670,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10373
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705365.212, "dur": 1.444,
    "args": {
      "External id": 2482671,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10374
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705376.277, "dur": 3.420,
    "args": {
      "External id": 2482672,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10375
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705378.444, "dur": 0.441,
    "args": {
      "External id": 2482673,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "57344"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10376
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705438.605, "dur": 49.109,
    "args": {
      "External id": 2482674,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 10377
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705492.357, "dur": 4.880,
    "args": {
      "External id": 2482675,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10378
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705494.879, "dur": 1.245,
    "args": {
      "External id": 2482676,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "117440512"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10379
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705498.442, "dur": 22.671,
    "args": {
      "External id": 2482677,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 10380
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705525.442, "dur": 24.622,
    "args": {
      "External id": 2482678,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 10381
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705542.499, "dur": 6.618,
    "args": {
      "External id": 2482679,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 10382
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705547.413, "dur": 1.354,
    "args": {
      "External id": 2482680,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 10383
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705553.467, "dur": 49.645,
    "args": {
      "External id": 2482681,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10384
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705554.569, "dur": 47.817,
    "args": {
      "External id": 2482682,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10385
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705607.241, "dur": 14.842,
    "args": {
      "External id": 2482683,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 10386
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705627.159, "dur": 24.142,
    "args": {
      "External id": 2482684,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", ""], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 10387
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705629.366, "dur": 21.419,
    "args": {
      "External id": 2482685,"Record function id": 0, "Concrete Inputs": ["", "[]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10388
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705634.506, "dur": 2.615,
    "args": {
      "External id": 2482686,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10389
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705657.271, "dur": 61.860,
    "args": {
      "External id": 2482687,"Record function id": 0, "Concrete Inputs": ["", "", "15", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], []], "Ev Idx": 10390
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705659.074, "dur": 59.618,
    "args": {
      "External id": 2482688,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], [], []], "Ev Idx": 10391
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705698.842, "dur": 4.678,
    "args": {
      "External id": 2482689,"Record function id": 0, "Concrete Inputs": ["[32000, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10392
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705704.894, "dur": 13.322,
    "args": {
      "External id": 2482690,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "float", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 10393
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705731.381, "dur": 4.410,
    "args": {
      "External id": 2482691,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 10394
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705732.981, "dur": 2.553,
    "args": {
      "External id": 2482692,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 10395
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705736.808, "dur": 0.988,
    "args": {
      "External id": 2482693,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 10396
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705737.130, "dur": 0.466,
    "args": {
      "External id": 2482694,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 10397
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705777.039, "dur": 21.025,
    "args": {
      "External id": 2482695,"Sequence number": 24740648, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[1], [], []], "Ev Idx": 10398
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705799.942, "dur": 18.035,
    "args": {
      "External id": 2482696,"Sequence number": 24740649, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[1], [], []], "Ev Idx": 10399
    }
  },
  {
    "ph": "s", "id": 2, "pid": 1336754, "tid": 1336754, "ts": 1514656705799.942,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward", "pid": 1336754, "tid": 1336754,
    "ts": 1514656705918.944, "dur": 39.303,
    "args": {
      "External id": 2482697,"Record function id": 0, "Ev Idx": 10400
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::div", "pid": 1336754, "tid": 1336754,
    "ts": 1514656706103.900, "dur": 39.914,
    "args": {
      "External id": 2482698,"Sequence number": 24740650, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "long int"], "Input Strides": [[1], []], "Input Dims": [[1], []], "Ev Idx": 10401
    }
  },
  {
    "ph": "s", "id": 1, "pid": 1336754, "tid": 1336754, "ts": 1514656706103.900,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::ones_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656706202.739, "dur": 25.763,
    "args": {
      "External id": 2482699,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "1"], "Input type": ["float", "", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[1], [], [], [], [], []], "Ev Idx": 10402
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514656706204.182, "dur": 8.176,
    "args": {
      "External id": 2482700,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "1"], "Input type": ["float", "", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[1], [], [], [], [], []], "Ev Idx": 10403
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514656706207.419, "dur": 4.332,
    "args": {
      "External id": 2482701,"Record function id": 0, "Concrete Inputs": ["[1]", "[1]", "6", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10404
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336754, "tid": 1336754,
    "ts": 1514656706213.934, "dur": 14.215,
    "args": {
      "External id": 2482702,"Record function id": 0, "Concrete Inputs": ["", "1."], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[1], []], "Ev Idx": 10405
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::div", "pid": 1336754, "tid": 1336754,
    "ts": 1514658027327.222, "dur": 81.133,
    "args": {
      "External id": 2482703,"Sequence number": 24740651, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "long int"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10406
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::div", "pid": 1336754, "tid": 1336754,
    "ts": 1514658027421.962, "dur": 23.376,
    "args": {
      "External id": 2482704,"Sequence number": 24740652, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "long int"], "Input Strides": [[1], []], "Input Dims": [[1], []], "Ev Idx": 10407
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336754, "tid": 1336754,
    "ts": 1514658027453.313, "dur": 23.345,
    "args": {
      "External id": 2482705,"Sequence number": 24740653, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "long int", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[1], [], []], "Ev Idx": 10408
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336754, "tid": 1336754,
    "ts": 1514658028074.945, "dur": 30.149,
    "args": {
      "External id": 2482706,"Sequence number": 24740654, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "long int", "Scalar"], "Input Strides": [[], [], []], "Input Dims": [[], [], []], "Ev Idx": 10409
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336754, "tid": 1336754,
    "ts": 1514658028686.319, "dur": 32.343,
    "args": {
      "External id": 2482707,"Sequence number": 24740655, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "long int", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[1], [], []], "Ev Idx": 10410
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_norm", "pid": 1336754, "tid": 1336754,
    "ts": 1514658030421.513, "dur": 3097.682,
    "args": {
      "External id": 2482708,"Record function id": 0, "Concrete Inputs": ["", "2.", ""], "Input type": ["TensorList", "Scalar", ""], "Input Strides": [[], [], []], "Input Dims": [[], [], []], "Ev Idx": 10411
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_norm", "pid": 1336754, "tid": 1336754,
    "ts": 1514658031092.397, "dur": 975.428,
    "args": {
      "External id": 2482709,"Record function id": 0, "Concrete Inputs": ["", "2.", ""], "Input type": ["TensorList", "Scalar", ""], "Input Strides": [[], [], []], "Input Dims": [[], [], []], "Ev Idx": 10412
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336754, "tid": 1336754,
    "ts": 1514658031115.800, "dur": 65.777,
    "args": {
      "External id": 2482710,"Record function id": 0, "Concrete Inputs": ["[36375]", "6", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 10413
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514658031119.298, "dur": 12.434,
    "args": {
      "External id": 2482711,"Record function id": 0, "Concrete Inputs": ["[36375]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10414
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336754, "tid": 1336754,
    "ts": 1514658031134.649, "dur": 46.584,
    "args": {
      "External id": 2482712,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[36375]], "Ev Idx": 10415
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336754, "tid": 1336754,
    "ts": 1514658031136.521, "dur": 44.187,
    "args": {
      "External id": 2482713,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[36375], []], "Ev Idx": 10416
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033566.807, "dur": 3.568,
    "args": {
      "External id": 2482714,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10417
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033572.967, "dur": 0.482,
    "args": {
      "External id": 2482715,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10418
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033574.607, "dur": 0.425,
    "args": {
      "External id": 2482716,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10419
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033576.135, "dur": 0.363,
    "args": {
      "External id": 2482717,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10420
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033577.548, "dur": 1.454,
    "args": {
      "External id": 2482718,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10421
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033580.320, "dur": 0.367,
    "args": {
      "External id": 2482719,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10422
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033581.764, "dur": 0.332,
    "args": {
      "External id": 2482720,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10423
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033584.915, "dur": 0.225,
    "args": {
      "External id": 2482721,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10424
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033585.941, "dur": 0.227,
    "args": {
      "External id": 2482722,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10425
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033587.002, "dur": 0.326,
    "args": {
      "External id": 2482723,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10426
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033588.189, "dur": 0.400,
    "args": {
      "External id": 2482724,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10427
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033589.533, "dur": 0.354,
    "args": {
      "External id": 2482725,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10428
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033590.638, "dur": 0.512,
    "args": {
      "External id": 2482726,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10429
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033591.951, "dur": 0.311,
    "args": {
      "External id": 2482727,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10430
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033593.011, "dur": 0.357,
    "args": {
      "External id": 2482728,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10431
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033595.811, "dur": 0.211,
    "args": {
      "External id": 2482729,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10432
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033596.933, "dur": 0.227,
    "args": {
      "External id": 2482730,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10433
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033598.074, "dur": 0.308,
    "args": {
      "External id": 2482731,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10434
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033599.342, "dur": 0.229,
    "args": {
      "External id": 2482732,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10435
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033600.381, "dur": 0.223,
    "args": {
      "External id": 2482733,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10436
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033601.411, "dur": 0.210,
    "args": {
      "External id": 2482734,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10437
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033602.347, "dur": 0.213,
    "args": {
      "External id": 2482735,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10438
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033603.349, "dur": 0.213,
    "args": {
      "External id": 2482736,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10439
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033606.100, "dur": 0.240,
    "args": {
      "External id": 2482737,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10440
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033607.187, "dur": 0.213,
    "args": {
      "External id": 2482738,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10441
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033608.293, "dur": 0.242,
    "args": {
      "External id": 2482739,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10442
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033609.356, "dur": 0.219,
    "args": {
      "External id": 2482740,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10443
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033610.618, "dur": 0.245,
    "args": {
      "External id": 2482741,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10444
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033611.689, "dur": 0.242,
    "args": {
      "External id": 2482742,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10445
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033612.815, "dur": 0.225,
    "args": {
      "External id": 2482743,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10446
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033613.861, "dur": 0.212,
    "args": {
      "External id": 2482744,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10447
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033616.368, "dur": 0.217,
    "args": {
      "External id": 2482745,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10448
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033617.404, "dur": 0.316,
    "args": {
      "External id": 2482746,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10449
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033618.729, "dur": 0.211,
    "args": {
      "External id": 2482747,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10450
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033619.877, "dur": 0.250,
    "args": {
      "External id": 2482748,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10451
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033620.953, "dur": 0.338,
    "args": {
      "External id": 2482749,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10452
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033622.192, "dur": 0.220,
    "args": {
      "External id": 2482750,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10453
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033623.146, "dur": 0.332,
    "args": {
      "External id": 2482751,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10454
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033624.224, "dur": 0.342,
    "args": {
      "External id": 2482752,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10455
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033661.396, "dur": 0.779,
    "args": {
      "External id": 2482753,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10456
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033665.291, "dur": 0.340,
    "args": {
      "External id": 2482754,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10457
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033666.806, "dur": 0.323,
    "args": {
      "External id": 2482755,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10458
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033667.985, "dur": 0.286,
    "args": {
      "External id": 2482756,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10459
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033669.122, "dur": 0.345,
    "args": {
      "External id": 2482757,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10460
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033670.357, "dur": 0.317,
    "args": {
      "External id": 2482758,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10461
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033673.018, "dur": 0.423,
    "args": {
      "External id": 2482759,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10462
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033674.353, "dur": 0.347,
    "args": {
      "External id": 2482760,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10463
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033677.336, "dur": 0.212,
    "args": {
      "External id": 2482761,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10464
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033678.513, "dur": 0.311,
    "args": {
      "External id": 2482762,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10465
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033679.793, "dur": 0.225,
    "args": {
      "External id": 2482763,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10466
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033680.904, "dur": 0.226,
    "args": {
      "External id": 2482764,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10467
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033681.973, "dur": 0.383,
    "args": {
      "External id": 2482765,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10468
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033683.220, "dur": 0.358,
    "args": {
      "External id": 2482766,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10469
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033684.518, "dur": 0.220,
    "args": {
      "External id": 2482767,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10470
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033685.491, "dur": 0.219,
    "args": {
      "External id": 2482768,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10471
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033688.103, "dur": 0.241,
    "args": {
      "External id": 2482769,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10472
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033689.138, "dur": 0.221,
    "args": {
      "External id": 2482770,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10473
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033690.142, "dur": 0.238,
    "args": {
      "External id": 2482771,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10474
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033691.140, "dur": 0.378,
    "args": {
      "External id": 2482772,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10475
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033692.305, "dur": 0.397,
    "args": {
      "External id": 2482773,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10476
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033693.541, "dur": 0.374,
    "args": {
      "External id": 2482774,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10477
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033694.744, "dur": 0.219,
    "args": {
      "External id": 2482775,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10478
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033695.813, "dur": 0.397,
    "args": {
      "External id": 2482776,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10479
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033698.489, "dur": 0.237,
    "args": {
      "External id": 2482777,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10480
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033699.449, "dur": 0.317,
    "args": {
      "External id": 2482778,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10481
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033700.627, "dur": 0.554,
    "args": {
      "External id": 2482779,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10482
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033701.919, "dur": 0.370,
    "args": {
      "External id": 2482780,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10483
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033703.042, "dur": 0.354,
    "args": {
      "External id": 2482781,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10484
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033704.166, "dur": 0.347,
    "args": {
      "External id": 2482782,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10485
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033705.372, "dur": 0.363,
    "args": {
      "External id": 2482783,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10486
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033706.464, "dur": 0.318,
    "args": {
      "External id": 2482784,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10487
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033709.007, "dur": 0.223,
    "args": {
      "External id": 2482785,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10488
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033710.112, "dur": 0.349,
    "args": {
      "External id": 2482786,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10489
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033711.288, "dur": 0.374,
    "args": {
      "External id": 2482787,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10490
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033712.540, "dur": 0.322,
    "args": {
      "External id": 2482788,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10491
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033713.708, "dur": 0.241,
    "args": {
      "External id": 2482789,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10492
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033714.814, "dur": 0.214,
    "args": {
      "External id": 2482790,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10493
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033716.537, "dur": 0.222,
    "args": {
      "External id": 2482791,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10494
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033717.631, "dur": 0.216,
    "args": {
      "External id": 2482792,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10495
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033719.895, "dur": 0.227,
    "args": {
      "External id": 2482793,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10496
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033720.868, "dur": 0.213,
    "args": {
      "External id": 2482794,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10497
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033721.795, "dur": 0.213,
    "args": {
      "External id": 2482795,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10498
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033722.766, "dur": 0.226,
    "args": {
      "External id": 2482796,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10499
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033723.781, "dur": 0.238,
    "args": {
      "External id": 2482797,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10500
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033724.771, "dur": 0.229,
    "args": {
      "External id": 2482798,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10501
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033725.764, "dur": 0.242,
    "args": {
      "External id": 2482799,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10502
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033726.774, "dur": 0.214,
    "args": {
      "External id": 2482800,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10503
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033729.198, "dur": 0.230,
    "args": {
      "External id": 2482801,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10504
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033730.252, "dur": 0.221,
    "args": {
      "External id": 2482802,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10505
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033731.227, "dur": 0.228,
    "args": {
      "External id": 2482803,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10506
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033732.229, "dur": 0.212,
    "args": {
      "External id": 2482804,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10507
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033733.257, "dur": 0.231,
    "args": {
      "External id": 2482805,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10508
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033734.233, "dur": 0.221,
    "args": {
      "External id": 2482806,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10509
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033735.945, "dur": 0.219,
    "args": {
      "External id": 2482807,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10510
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033736.990, "dur": 0.211,
    "args": {
      "External id": 2482808,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10511
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033739.500, "dur": 0.218,
    "args": {
      "External id": 2482809,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10512
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033740.427, "dur": 0.231,
    "args": {
      "External id": 2482810,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10513
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033741.426, "dur": 0.208,
    "args": {
      "External id": 2482811,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10514
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033742.352, "dur": 0.218,
    "args": {
      "External id": 2482812,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10515
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033743.301, "dur": 0.223,
    "args": {
      "External id": 2482813,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10516
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033744.287, "dur": 0.223,
    "args": {
      "External id": 2482814,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10517
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033745.260, "dur": 0.227,
    "args": {
      "External id": 2482815,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10518
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033746.281, "dur": 0.238,
    "args": {
      "External id": 2482816,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10519
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033749.078, "dur": 0.235,
    "args": {
      "External id": 2482817,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10520
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033750.046, "dur": 0.240,
    "args": {
      "External id": 2482818,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10521
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033751.066, "dur": 0.222,
    "args": {
      "External id": 2482819,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10522
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033752.084, "dur": 0.201,
    "args": {
      "External id": 2482820,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10523
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033753.070, "dur": 0.236,
    "args": {
      "External id": 2482821,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10524
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033754.058, "dur": 0.219,
    "args": {
      "External id": 2482822,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10525
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033755.504, "dur": 0.242,
    "args": {
      "External id": 2482823,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10526
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033756.595, "dur": 0.220,
    "args": {
      "External id": 2482824,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10527
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033759.072, "dur": 0.207,
    "args": {
      "External id": 2482825,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10528
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033760.123, "dur": 0.240,
    "args": {
      "External id": 2482826,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10529
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033761.254, "dur": 0.218,
    "args": {
      "External id": 2482827,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10530
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033762.210, "dur": 0.222,
    "args": {
      "External id": 2482828,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10531
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033763.287, "dur": 0.221,
    "args": {
      "External id": 2482829,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10532
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033764.381, "dur": 0.364,
    "args": {
      "External id": 2482830,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10533
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033765.739, "dur": 0.215,
    "args": {
      "External id": 2482831,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10534
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033766.779, "dur": 0.372,
    "args": {
      "External id": 2482832,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10535
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033769.651, "dur": 0.353,
    "args": {
      "External id": 2482833,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10536
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033770.734, "dur": 0.358,
    "args": {
      "External id": 2482834,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10537
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033771.794, "dur": 0.329,
    "args": {
      "External id": 2482835,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10538
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033772.823, "dur": 0.358,
    "args": {
      "External id": 2482836,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10539
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033773.895, "dur": 0.368,
    "args": {
      "External id": 2482837,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10540
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033775.043, "dur": 0.321,
    "args": {
      "External id": 2482838,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10541
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033776.104, "dur": 0.335,
    "args": {
      "External id": 2482839,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10542
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033777.148, "dur": 0.372,
    "args": {
      "External id": 2482840,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10543
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033779.874, "dur": 0.228,
    "args": {
      "External id": 2482841,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10544
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033780.821, "dur": 0.335,
    "args": {
      "External id": 2482842,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10545
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033782.333, "dur": 0.323,
    "args": {
      "External id": 2482843,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10546
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033783.379, "dur": 0.240,
    "args": {
      "External id": 2482844,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10547
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033784.345, "dur": 0.211,
    "args": {
      "External id": 2482845,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10548
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033785.321, "dur": 0.221,
    "args": {
      "External id": 2482846,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10549
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033786.409, "dur": 0.216,
    "args": {
      "External id": 2482847,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10550
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033787.344, "dur": 0.213,
    "args": {
      "External id": 2482848,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10551
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033789.736, "dur": 0.240,
    "args": {
      "External id": 2482849,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10552
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033790.834, "dur": 0.214,
    "args": {
      "External id": 2482850,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10553
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033791.751, "dur": 0.366,
    "args": {
      "External id": 2482851,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10554
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033792.932, "dur": 0.385,
    "args": {
      "External id": 2482852,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10555
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033794.109, "dur": 0.383,
    "args": {
      "External id": 2482853,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10556
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033795.257, "dur": 0.374,
    "args": {
      "External id": 2482854,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10557
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033796.471, "dur": 0.382,
    "args": {
      "External id": 2482855,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10558
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033797.664, "dur": 0.382,
    "args": {
      "External id": 2482856,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10559
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033800.422, "dur": 0.210,
    "args": {
      "External id": 2482857,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10560
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033801.480, "dur": 0.250,
    "args": {
      "External id": 2482858,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10561
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033802.450, "dur": 0.354,
    "args": {
      "External id": 2482859,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10562
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033803.510, "dur": 0.244,
    "args": {
      "External id": 2482860,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10563
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033804.529, "dur": 0.221,
    "args": {
      "External id": 2482861,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10564
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033805.658, "dur": 0.228,
    "args": {
      "External id": 2482862,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10565
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033807.267, "dur": 0.227,
    "args": {
      "External id": 2482863,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10566
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033808.353, "dur": 0.241,
    "args": {
      "External id": 2482864,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10567
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033810.788, "dur": 0.237,
    "args": {
      "External id": 2482865,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10568
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033811.892, "dur": 0.216,
    "args": {
      "External id": 2482866,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10569
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033812.957, "dur": 0.209,
    "args": {
      "External id": 2482867,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10570
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033814.061, "dur": 0.213,
    "args": {
      "External id": 2482868,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10571
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033815.048, "dur": 0.209,
    "args": {
      "External id": 2482869,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10572
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033815.997, "dur": 0.245,
    "args": {
      "External id": 2482870,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10573
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033817.231, "dur": 0.226,
    "args": {
      "External id": 2482871,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10574
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033818.294, "dur": 0.238,
    "args": {
      "External id": 2482872,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10575
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033822.032, "dur": 0.252,
    "args": {
      "External id": 2482873,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10576
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033823.146, "dur": 0.206,
    "args": {
      "External id": 2482874,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10577
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033824.220, "dur": 0.208,
    "args": {
      "External id": 2482875,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10578
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033825.176, "dur": 0.220,
    "args": {
      "External id": 2482876,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10579
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033826.156, "dur": 0.220,
    "args": {
      "External id": 2482877,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10580
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033827.155, "dur": 0.248,
    "args": {
      "External id": 2482878,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10581
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033828.228, "dur": 0.210,
    "args": {
      "External id": 2482879,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10582
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033829.176, "dur": 0.233,
    "args": {
      "External id": 2482880,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10583
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033831.495, "dur": 0.214,
    "args": {
      "External id": 2482881,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10584
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033832.460, "dur": 0.216,
    "args": {
      "External id": 2482882,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10585
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033833.396, "dur": 0.233,
    "args": {
      "External id": 2482883,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10586
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033834.401, "dur": 0.234,
    "args": {
      "External id": 2482884,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10587
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033835.385, "dur": 0.235,
    "args": {
      "External id": 2482885,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10588
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033836.335, "dur": 0.217,
    "args": {
      "External id": 2482886,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10589
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033837.580, "dur": 0.223,
    "args": {
      "External id": 2482887,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10590
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033838.523, "dur": 0.246,
    "args": {
      "External id": 2482888,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10591
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033841.183, "dur": 0.213,
    "args": {
      "External id": 2482889,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10592
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033842.191, "dur": 0.213,
    "args": {
      "External id": 2482890,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10593
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033843.219, "dur": 0.217,
    "args": {
      "External id": 2482891,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10594
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033844.155, "dur": 0.237,
    "args": {
      "External id": 2482892,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10595
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033845.156, "dur": 0.213,
    "args": {
      "External id": 2482893,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10596
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033846.129, "dur": 0.232,
    "args": {
      "External id": 2482894,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10597
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033847.106, "dur": 0.233,
    "args": {
      "External id": 2482895,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10598
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033848.078, "dur": 0.217,
    "args": {
      "External id": 2482896,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10599
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033850.438, "dur": 0.218,
    "args": {
      "External id": 2482897,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10600
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033851.416, "dur": 0.224,
    "args": {
      "External id": 2482898,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10601
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033852.387, "dur": 0.336,
    "args": {
      "External id": 2482899,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10602
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033853.573, "dur": 0.350,
    "args": {
      "External id": 2482900,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10603
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033854.665, "dur": 0.518,
    "args": {
      "External id": 2482901,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10604
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033855.911, "dur": 0.352,
    "args": {
      "External id": 2482902,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10605
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033856.992, "dur": 0.343,
    "args": {
      "External id": 2482903,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10606
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033858.123, "dur": 0.498,
    "args": {
      "External id": 2482904,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10607
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033860.808, "dur": 0.222,
    "args": {
      "External id": 2482905,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10608
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033861.902, "dur": 0.213,
    "args": {
      "External id": 2482906,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10609
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033862.844, "dur": 0.313,
    "args": {
      "External id": 2482907,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10610
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033864.147, "dur": 0.236,
    "args": {
      "External id": 2482908,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10611
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033865.097, "dur": 0.224,
    "args": {
      "External id": 2482909,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10612
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033866.072, "dur": 0.221,
    "args": {
      "External id": 2482910,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10613
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033867.037, "dur": 0.211,
    "args": {
      "External id": 2482911,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10614
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033868.069, "dur": 0.214,
    "args": {
      "External id": 2482912,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10615
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033870.259, "dur": 0.216,
    "args": {
      "External id": 2482913,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10616
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033871.439, "dur": 0.221,
    "args": {
      "External id": 2482914,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10617
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033873.087, "dur": 0.304,
    "args": {
      "External id": 2482915,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10618
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033874.260, "dur": 0.215,
    "args": {
      "External id": 2482916,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10619
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033875.297, "dur": 0.237,
    "args": {
      "External id": 2482917,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10620
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033876.354, "dur": 0.240,
    "args": {
      "External id": 2482918,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10621
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033877.461, "dur": 0.218,
    "args": {
      "External id": 2482919,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10622
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033878.412, "dur": 0.215,
    "args": {
      "External id": 2482920,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10623
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033881.121, "dur": 0.242,
    "args": {
      "External id": 2482921,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10624
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033882.168, "dur": 0.233,
    "args": {
      "External id": 2482922,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10625
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033883.199, "dur": 0.211,
    "args": {
      "External id": 2482923,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10626
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033884.184, "dur": 0.219,
    "args": {
      "External id": 2482924,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10627
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033885.121, "dur": 0.739,
    "args": {
      "External id": 2482925,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10628
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033886.682, "dur": 0.401,
    "args": {
      "External id": 2482926,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10629
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033887.880, "dur": 0.319,
    "args": {
      "External id": 2482927,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10630
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033888.960, "dur": 0.334,
    "args": {
      "External id": 2482928,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10631
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033891.466, "dur": 0.217,
    "args": {
      "External id": 2482929,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10632
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033892.468, "dur": 0.375,
    "args": {
      "External id": 2482930,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10633
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033893.713, "dur": 0.180,
    "args": {
      "External id": 2482931,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10634
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033894.659, "dur": 0.354,
    "args": {
      "External id": 2482932,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10635
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033895.759, "dur": 0.214,
    "args": {
      "External id": 2482933,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10636
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033896.737, "dur": 0.229,
    "args": {
      "External id": 2482934,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10637
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033897.686, "dur": 0.218,
    "args": {
      "External id": 2482935,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10638
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033898.683, "dur": 0.217,
    "args": {
      "External id": 2482936,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10639
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033901.269, "dur": 0.223,
    "args": {
      "External id": 2482937,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10640
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033902.211, "dur": 0.223,
    "args": {
      "External id": 2482938,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10641
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033903.197, "dur": 0.215,
    "args": {
      "External id": 2482939,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10642
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033904.173, "dur": 0.227,
    "args": {
      "External id": 2482940,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10643
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033905.123, "dur": 0.240,
    "args": {
      "External id": 2482941,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10644
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033906.113, "dur": 0.212,
    "args": {
      "External id": 2482942,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10645
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033907.041, "dur": 0.218,
    "args": {
      "External id": 2482943,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10646
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033907.971, "dur": 0.220,
    "args": {
      "External id": 2482944,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10647
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033910.462, "dur": 0.241,
    "args": {
      "External id": 2482945,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10648
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033911.465, "dur": 0.236,
    "args": {
      "External id": 2482946,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10649
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033912.938, "dur": 0.225,
    "args": {
      "External id": 2482947,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10650
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033913.983, "dur": 0.212,
    "args": {
      "External id": 2482948,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10651
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033914.993, "dur": 0.214,
    "args": {
      "External id": 2482949,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10652
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033915.927, "dur": 0.250,
    "args": {
      "External id": 2482950,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10653
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033916.932, "dur": 0.233,
    "args": {
      "External id": 2482951,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10654
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033917.918, "dur": 0.214,
    "args": {
      "External id": 2482952,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10655
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033920.370, "dur": 0.219,
    "args": {
      "External id": 2482953,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10656
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033921.476, "dur": 0.224,
    "args": {
      "External id": 2482954,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10657
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033922.693, "dur": 0.216,
    "args": {
      "External id": 2482955,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10658
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033923.691, "dur": 0.229,
    "args": {
      "External id": 2482956,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10659
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033924.763, "dur": 0.228,
    "args": {
      "External id": 2482957,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10660
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033925.887, "dur": 0.214,
    "args": {
      "External id": 2482958,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10661
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033926.952, "dur": 0.232,
    "args": {
      "External id": 2482959,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10662
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033928.095, "dur": 0.210,
    "args": {
      "External id": 2482960,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10663
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033930.447, "dur": 0.216,
    "args": {
      "External id": 2482961,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10664
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033931.454, "dur": 0.219,
    "args": {
      "External id": 2482962,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10665
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033932.394, "dur": 0.229,
    "args": {
      "External id": 2482963,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10666
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033933.418, "dur": 0.227,
    "args": {
      "External id": 2482964,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10667
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033934.437, "dur": 0.214,
    "args": {
      "External id": 2482965,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10668
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033935.402, "dur": 0.215,
    "args": {
      "External id": 2482966,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10669
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033936.409, "dur": 0.229,
    "args": {
      "External id": 2482967,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10670
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033937.410, "dur": 0.242,
    "args": {
      "External id": 2482968,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10671
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033940.021, "dur": 0.216,
    "args": {
      "External id": 2482969,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10672
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033944.496, "dur": 0.220,
    "args": {
      "External id": 2482970,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10673
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033945.486, "dur": 0.279,
    "args": {
      "External id": 2482971,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10674
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033946.527, "dur": 0.211,
    "args": {
      "External id": 2482972,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10675
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033947.466, "dur": 0.351,
    "args": {
      "External id": 2482973,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10676
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033948.644, "dur": 0.375,
    "args": {
      "External id": 2482974,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10677
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033949.803, "dur": 0.238,
    "args": {
      "External id": 2482975,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10678
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033950.776, "dur": 0.347,
    "args": {
      "External id": 2482976,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10679
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033953.480, "dur": 0.345,
    "args": {
      "External id": 2482977,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10680
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033954.676, "dur": 0.328,
    "args": {
      "External id": 2482978,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10681
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033955.739, "dur": 0.343,
    "args": {
      "External id": 2482979,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10682
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033956.789, "dur": 0.342,
    "args": {
      "External id": 2482980,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10683
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033957.856, "dur": 0.358,
    "args": {
      "External id": 2482981,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10684
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033958.964, "dur": 0.362,
    "args": {
      "External id": 2482982,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10685
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033960.669, "dur": 0.354,
    "args": {
      "External id": 2482983,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10686
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033961.762, "dur": 0.340,
    "args": {
      "External id": 2482984,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10687
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033964.513, "dur": 0.231,
    "args": {
      "External id": 2482985,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10688
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033965.609, "dur": 0.231,
    "args": {
      "External id": 2482986,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10689
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033967.924, "dur": 0.230,
    "args": {
      "External id": 2482987,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10690
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033968.902, "dur": 0.211,
    "args": {
      "External id": 2482988,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10691
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033969.847, "dur": 0.215,
    "args": {
      "External id": 2482989,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10692
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033970.796, "dur": 0.230,
    "args": {
      "External id": 2482990,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10693
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033971.763, "dur": 0.223,
    "args": {
      "External id": 2482991,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10694
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033972.816, "dur": 0.219,
    "args": {
      "External id": 2482992,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10695
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033975.444, "dur": 0.211,
    "args": {
      "External id": 2482993,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10696
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033976.487, "dur": 0.237,
    "args": {
      "External id": 2482994,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10697
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033977.554, "dur": 0.409,
    "args": {
      "External id": 2482995,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10698
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033978.830, "dur": 0.419,
    "args": {
      "External id": 2482996,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10699
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033980.141, "dur": 0.228,
    "args": {
      "External id": 2482997,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10700
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658033981.218, "dur": 0.480,
    "args": {
      "External id": 2482998,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10701
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034010.160, "dur": 0.637,
    "args": {
      "External id": 2482999,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10702
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034012.527, "dur": 0.365,
    "args": {
      "External id": 2483000,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10703
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034015.633, "dur": 0.350,
    "args": {
      "External id": 2483001,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10704
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034017.060, "dur": 0.372,
    "args": {
      "External id": 2483002,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10705
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034018.284, "dur": 0.363,
    "args": {
      "External id": 2483003,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10706
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034019.620, "dur": 0.530,
    "args": {
      "External id": 2483004,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10707
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::stack", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034078.127, "dur": 1508.203,
    "args": {
      "External id": 2483005,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["TensorList", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10708
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::stack", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034518.206, "dur": 978.155,
    "args": {
      "External id": 2483006,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["TensorList", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10709
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034541.223, "dur": 7.969,
    "args": {
      "External id": 2483007,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10710
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034545.404, "dur": 3.178,
    "args": {
      "External id": 2483008,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10711
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034549.961, "dur": 3.112,
    "args": {
      "External id": 2483009,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10712
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034551.227, "dur": 1.739,
    "args": {
      "External id": 2483010,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10713
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034553.538, "dur": 3.066,
    "args": {
      "External id": 2483011,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10714
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034555.696, "dur": 0.818,
    "args": {
      "External id": 2483012,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10715
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034556.978, "dur": 2.105,
    "args": {
      "External id": 2483013,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10716
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034557.802, "dur": 1.218,
    "args": {
      "External id": 2483014,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10717
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034560.923, "dur": 4.337,
    "args": {
      "External id": 2483015,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10718
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034564.634, "dur": 0.551,
    "args": {
      "External id": 2483016,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10719
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034565.705, "dur": 1.499,
    "args": {
      "External id": 2483017,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10720
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034566.391, "dur": 0.726,
    "args": {
      "External id": 2483018,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10721
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034567.564, "dur": 1.397,
    "args": {
      "External id": 2483019,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10722
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034568.215, "dur": 0.678,
    "args": {
      "External id": 2483020,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10723
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034569.297, "dur": 3.290,
    "args": {
      "External id": 2483021,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10724
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034571.636, "dur": 0.884,
    "args": {
      "External id": 2483022,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10725
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034572.938, "dur": 1.078,
    "args": {
      "External id": 2483023,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10726
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034573.512, "dur": 0.428,
    "args": {
      "External id": 2483024,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10727
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034574.406, "dur": 2.671,
    "args": {
      "External id": 2483025,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10728
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034574.920, "dur": 1.955,
    "args": {
      "External id": 2483026,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10729
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034577.445, "dur": 2.338,
    "args": {
      "External id": 2483027,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10730
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034579.169, "dur": 0.543,
    "args": {
      "External id": 2483028,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10731
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034580.124, "dur": 1.023,
    "args": {
      "External id": 2483029,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10732
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034580.538, "dur": 0.536,
    "args": {
      "External id": 2483030,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10733
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034582.802, "dur": 3.808,
    "args": {
      "External id": 2483031,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10734
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034586.132, "dur": 0.411,
    "args": {
      "External id": 2483032,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10735
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034586.863, "dur": 1.096,
    "args": {
      "External id": 2483033,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10736
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034587.259, "dur": 0.634,
    "args": {
      "External id": 2483034,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10737
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034588.325, "dur": 1.380,
    "args": {
      "External id": 2483035,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10738
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034588.959, "dur": 0.673,
    "args": {
      "External id": 2483036,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10739
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034590.125, "dur": 3.931,
    "args": {
      "External id": 2483037,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10740
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034593.418, "dur": 0.572,
    "args": {
      "External id": 2483038,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10741
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034594.389, "dur": 1.011,
    "args": {
      "External id": 2483039,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10742
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034594.899, "dur": 0.435,
    "args": {
      "External id": 2483040,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10743
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034595.824, "dur": 2.423,
    "args": {
      "External id": 2483041,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10744
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034596.426, "dur": 1.631,
    "args": {
      "External id": 2483042,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10745
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034598.581, "dur": 2.721,
    "args": {
      "External id": 2483043,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10746
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034600.776, "dur": 0.458,
    "args": {
      "External id": 2483044,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10747
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034601.636, "dur": 1.149,
    "args": {
      "External id": 2483045,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10748
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034602.040, "dur": 0.678,
    "args": {
      "External id": 2483046,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10749
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034604.634, "dur": 3.911,
    "args": {
      "External id": 2483047,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10750
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034607.958, "dur": 0.524,
    "args": {
      "External id": 2483048,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10751
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034608.824, "dur": 1.314,
    "args": {
      "External id": 2483049,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10752
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034609.467, "dur": 0.604,
    "args": {
      "External id": 2483050,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10753
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034610.564, "dur": 1.434,
    "args": {
      "External id": 2483051,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10754
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034611.193, "dur": 0.743,
    "args": {
      "External id": 2483052,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10755
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034612.538, "dur": 3.378,
    "args": {
      "External id": 2483053,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10756
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034614.996, "dur": 0.858,
    "args": {
      "External id": 2483054,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10757
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034616.292, "dur": 1.573,
    "args": {
      "External id": 2483055,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10758
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034616.770, "dur": 1.029,
    "args": {
      "External id": 2483056,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10759
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034618.087, "dur": 2.208,
    "args": {
      "External id": 2483057,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10760
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034618.451, "dur": 1.681,
    "args": {
      "External id": 2483058,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10761
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034620.568, "dur": 2.618,
    "args": {
      "External id": 2483059,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10762
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034622.543, "dur": 0.576,
    "args": {
      "External id": 2483060,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10763
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034623.496, "dur": 1.094,
    "args": {
      "External id": 2483061,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10764
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034623.860, "dur": 0.660,
    "args": {
      "External id": 2483062,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10765
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034626.320, "dur": 3.871,
    "args": {
      "External id": 2483063,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10766
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034629.690, "dur": 0.434,
    "args": {
      "External id": 2483064,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10767
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034630.619, "dur": 1.340,
    "args": {
      "External id": 2483065,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10768
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034631.258, "dur": 0.634,
    "args": {
      "External id": 2483066,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10769
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034632.284, "dur": 1.367,
    "args": {
      "External id": 2483067,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10770
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034632.927, "dur": 0.660,
    "args": {
      "External id": 2483068,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10771
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034633.978, "dur": 3.163,
    "args": {
      "External id": 2483069,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10772
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034636.459, "dur": 0.614,
    "args": {
      "External id": 2483070,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10773
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034637.482, "dur": 0.861,
    "args": {
      "External id": 2483071,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10774
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034637.871, "dur": 0.405,
    "args": {
      "External id": 2483072,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10775
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034638.660, "dur": 2.758,
    "args": {
      "External id": 2483073,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10776
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034639.156, "dur": 2.192,
    "args": {
      "External id": 2483074,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10777
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034641.643, "dur": 4.355,
    "args": {
      "External id": 2483075,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10778
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034645.327, "dur": 0.604,
    "args": {
      "External id": 2483076,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10779
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034646.528, "dur": 2.626,
    "args": {
      "External id": 2483077,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10780
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034647.104, "dur": 1.973,
    "args": {
      "External id": 2483078,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10781
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034650.884, "dur": 2.198,
    "args": {
      "External id": 2483079,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10782
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034652.456, "dur": 0.557,
    "args": {
      "External id": 2483080,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10783
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034653.431, "dur": 1.360,
    "args": {
      "External id": 2483081,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10784
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034654.037, "dur": 0.692,
    "args": {
      "External id": 2483082,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10785
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034655.203, "dur": 3.105,
    "args": {
      "External id": 2483083,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10786
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034657.311, "dur": 0.924,
    "args": {
      "External id": 2483084,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10787
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034658.699, "dur": 2.827,
    "args": {
      "External id": 2483085,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10788
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034660.738, "dur": 0.717,
    "args": {
      "External id": 2483086,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10789
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034661.942, "dur": 1.220,
    "args": {
      "External id": 2483087,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10790
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034662.447, "dur": 0.650,
    "args": {
      "External id": 2483088,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10791
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034663.486, "dur": 2.667,
    "args": {
      "External id": 2483089,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10792
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034665.499, "dur": 0.583,
    "args": {
      "External id": 2483090,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10793
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034666.518, "dur": 2.422,
    "args": {
      "External id": 2483091,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10794
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034668.313, "dur": 0.559,
    "args": {
      "External id": 2483092,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10795
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034669.160, "dur": 2.593,
    "args": {
      "External id": 2483093,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10796
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034669.708, "dur": 1.978,
    "args": {
      "External id": 2483094,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10797
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034673.458, "dur": 2.035,
    "args": {
      "External id": 2483095,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10798
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034674.955, "dur": 0.460,
    "args": {
      "External id": 2483096,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10799
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034675.834, "dur": 1.579,
    "args": {
      "External id": 2483097,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10800
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034676.322, "dur": 1.019,
    "args": {
      "External id": 2483098,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10801
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034678.101, "dur": 3.433,
    "args": {
      "External id": 2483099,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10802
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034680.607, "dur": 0.851,
    "args": {
      "External id": 2483100,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10803
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034681.888, "dur": 2.194,
    "args": {
      "External id": 2483101,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10804
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034683.298, "dur": 0.709,
    "args": {
      "External id": 2483102,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10805
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034684.303, "dur": 1.290,
    "args": {
      "External id": 2483103,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10806
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034684.824, "dur": 0.698,
    "args": {
      "External id": 2483104,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10807
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034685.919, "dur": 2.819,
    "args": {
      "External id": 2483105,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10808
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034688.041, "dur": 0.633,
    "args": {
      "External id": 2483106,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10809
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034689.114, "dur": 2.486,
    "args": {
      "External id": 2483107,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10810
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034690.308, "dur": 1.225,
    "args": {
      "External id": 2483108,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10811
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034691.947, "dur": 3.100,
    "args": {
      "External id": 2483109,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10812
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034693.087, "dur": 1.893,
    "args": {
      "External id": 2483110,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10813
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034696.640, "dur": 2.062,
    "args": {
      "External id": 2483111,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10814
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034697.858, "dur": 0.777,
    "args": {
      "External id": 2483112,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10815
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034698.942, "dur": 1.792,
    "args": {
      "External id": 2483113,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10816
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034699.919, "dur": 0.749,
    "args": {
      "External id": 2483114,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10817
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034701.074, "dur": 3.239,
    "args": {
      "External id": 2483115,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10818
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034703.667, "dur": 0.579,
    "args": {
      "External id": 2483116,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10819
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034704.697, "dur": 1.847,
    "args": {
      "External id": 2483117,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10820
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034705.853, "dur": 0.625,
    "args": {
      "External id": 2483118,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10821
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034706.897, "dur": 1.616,
    "args": {
      "External id": 2483119,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10822
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034707.698, "dur": 0.751,
    "args": {
      "External id": 2483120,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10823
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034708.871, "dur": 3.414,
    "args": {
      "External id": 2483121,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10824
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034711.296, "dur": 0.922,
    "args": {
      "External id": 2483122,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10825
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034712.593, "dur": 1.361,
    "args": {
      "External id": 2483123,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10826
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034713.477, "dur": 0.411,
    "args": {
      "External id": 2483124,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10827
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034714.194, "dur": 3.024,
    "args": {
      "External id": 2483125,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10828
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034715.042, "dur": 1.976,
    "args": {
      "External id": 2483126,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10829
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034718.712, "dur": 1.354,
    "args": {
      "External id": 2483127,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10830
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034719.456, "dur": 0.543,
    "args": {
      "External id": 2483128,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10831
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034720.532, "dur": 1.461,
    "args": {
      "External id": 2483129,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10832
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034721.375, "dur": 0.550,
    "args": {
      "External id": 2483130,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10833
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034722.330, "dur": 3.308,
    "args": {
      "External id": 2483131,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10834
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034724.864, "dur": 0.711,
    "args": {
      "External id": 2483132,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10835
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034726.006, "dur": 1.887,
    "args": {
      "External id": 2483133,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10836
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034726.939, "dur": 0.892,
    "args": {
      "External id": 2483134,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10837
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034728.205, "dur": 1.689,
    "args": {
      "External id": 2483135,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10838
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034729.093, "dur": 0.730,
    "args": {
      "External id": 2483136,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10839
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034730.229, "dur": 2.964,
    "args": {
      "External id": 2483137,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10840
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034732.351, "dur": 0.775,
    "args": {
      "External id": 2483138,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10841
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034733.618, "dur": 1.623,
    "args": {
      "External id": 2483139,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10842
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034734.600, "dur": 0.577,
    "args": {
      "External id": 2483140,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10843
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034735.483, "dur": 2.930,
    "args": {
      "External id": 2483141,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10844
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034736.488, "dur": 1.858,
    "args": {
      "External id": 2483142,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10845
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034739.879, "dur": 1.688,
    "args": {
      "External id": 2483143,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10846
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034740.811, "dur": 0.690,
    "args": {
      "External id": 2483144,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10847
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034741.931, "dur": 1.942,
    "args": {
      "External id": 2483145,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10848
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034742.974, "dur": 0.837,
    "args": {
      "External id": 2483146,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10849
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034744.310, "dur": 3.541,
    "args": {
      "External id": 2483147,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10850
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034747.136, "dur": 0.648,
    "args": {
      "External id": 2483148,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10851
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034748.223, "dur": 1.854,
    "args": {
      "External id": 2483149,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10852
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034749.155, "dur": 0.860,
    "args": {
      "External id": 2483150,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10853
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034750.398, "dur": 1.724,
    "args": {
      "External id": 2483151,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10854
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034751.402, "dur": 0.645,
    "args": {
      "External id": 2483152,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10855
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034752.461, "dur": 2.374,
    "args": {
      "External id": 2483153,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10856
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034754.277, "dur": 0.480,
    "args": {
      "External id": 2483154,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10857
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034755.217, "dur": 1.613,
    "args": {
      "External id": 2483155,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10858
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034756.109, "dur": 0.650,
    "args": {
      "External id": 2483156,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10859
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034757.104, "dur": 3.029,
    "args": {
      "External id": 2483157,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10860
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034758.028, "dur": 2.037,
    "args": {
      "External id": 2483158,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10861
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034761.633, "dur": 1.797,
    "args": {
      "External id": 2483159,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10862
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034762.889, "dur": 0.475,
    "args": {
      "External id": 2483160,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10863
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034763.701, "dur": 1.316,
    "args": {
      "External id": 2483161,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10864
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034764.414, "dur": 0.536,
    "args": {
      "External id": 2483162,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10865
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034765.337, "dur": 3.103,
    "args": {
      "External id": 2483163,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10866
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034767.687, "dur": 0.682,
    "args": {
      "External id": 2483164,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10867
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034768.713, "dur": 1.695,
    "args": {
      "External id": 2483165,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10868
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034769.789, "dur": 0.553,
    "args": {
      "External id": 2483166,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10869
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034770.642, "dur": 1.539,
    "args": {
      "External id": 2483167,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10870
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034771.499, "dur": 0.615,
    "args": {
      "External id": 2483168,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10871
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034772.546, "dur": 2.751,
    "args": {
      "External id": 2483169,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10872
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034774.720, "dur": 0.510,
    "args": {
      "External id": 2483170,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10873
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034775.630, "dur": 1.323,
    "args": {
      "External id": 2483171,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10874
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034776.422, "dur": 0.459,
    "args": {
      "External id": 2483172,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10875
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034777.303, "dur": 3.067,
    "args": {
      "External id": 2483173,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10876
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034778.059, "dur": 2.057,
    "args": {
      "External id": 2483174,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10877
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034782.189, "dur": 1.601,
    "args": {
      "External id": 2483175,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10878
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034783.265, "dur": 0.457,
    "args": {
      "External id": 2483176,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10879
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034784.150, "dur": 1.587,
    "args": {
      "External id": 2483177,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10880
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034785.039, "dur": 0.631,
    "args": {
      "External id": 2483178,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10881
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034786.073, "dur": 3.349,
    "args": {
      "External id": 2483179,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10882
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034788.783, "dur": 0.569,
    "args": {
      "External id": 2483180,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10883
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034789.818, "dur": 1.610,
    "args": {
      "External id": 2483181,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10884
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034790.695, "dur": 0.668,
    "args": {
      "External id": 2483182,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10885
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034791.877, "dur": 1.373,
    "args": {
      "External id": 2483183,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10886
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034792.609, "dur": 0.577,
    "args": {
      "External id": 2483184,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10887
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034793.491, "dur": 2.604,
    "args": {
      "External id": 2483185,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10888
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034795.609, "dur": 0.417,
    "args": {
      "External id": 2483186,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10889
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034796.322, "dur": 1.533,
    "args": {
      "External id": 2483187,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10890
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034797.172, "dur": 0.523,
    "args": {
      "External id": 2483188,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10891
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034798.110, "dur": 2.326,
    "args": {
      "External id": 2483189,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10892
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034798.750, "dur": 1.613,
    "args": {
      "External id": 2483190,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10893
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034802.328, "dur": 1.911,
    "args": {
      "External id": 2483191,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10894
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034803.377, "dur": 0.568,
    "args": {
      "External id": 2483192,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10895
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034804.481, "dur": 1.274,
    "args": {
      "External id": 2483193,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10896
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034805.060, "dur": 0.629,
    "args": {
      "External id": 2483194,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10897
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034806.104, "dur": 3.187,
    "args": {
      "External id": 2483195,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10898
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034808.555, "dur": 0.662,
    "args": {
      "External id": 2483196,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10899
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034809.571, "dur": 1.408,
    "args": {
      "External id": 2483197,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10900
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034810.334, "dur": 0.580,
    "args": {
      "External id": 2483198,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10901
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034811.239, "dur": 1.729,
    "args": {
      "External id": 2483199,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10902
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034812.261, "dur": 0.642,
    "args": {
      "External id": 2483200,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10903
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034838.110, "dur": 3.170,
    "args": {
      "External id": 2489345,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10904
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034840.666, "dur": 0.450,
    "args": {
      "External id": 2489346,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10905
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034841.541, "dur": 1.403,
    "args": {
      "External id": 2489347,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10906
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034842.375, "dur": 0.506,
    "args": {
      "External id": 2489348,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10907
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034843.241, "dur": 2.207,
    "args": {
      "External id": 2489349,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10908
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034843.884, "dur": 1.495,
    "args": {
      "External id": 2489350,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10909
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034847.378, "dur": 1.305,
    "args": {
      "External id": 2489351,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10910
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034848.247, "dur": 0.372,
    "args": {
      "External id": 2489352,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10911
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034849.046, "dur": 1.091,
    "args": {
      "External id": 2489353,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10912
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034849.583, "dur": 0.394,
    "args": {
      "External id": 2489354,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10913
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034850.384, "dur": 3.006,
    "args": {
      "External id": 2489355,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10914
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034852.866, "dur": 0.457,
    "args": {
      "External id": 2489356,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10915
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034853.673, "dur": 1.553,
    "args": {
      "External id": 2489357,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10916
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034854.284, "dur": 0.782,
    "args": {
      "External id": 2489358,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10917
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034855.459, "dur": 1.784,
    "args": {
      "External id": 2489359,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10918
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034856.573, "dur": 0.606,
    "args": {
      "External id": 2489360,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10919
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034857.509, "dur": 2.459,
    "args": {
      "External id": 2489361,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10920
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034859.430, "dur": 0.475,
    "args": {
      "External id": 2489362,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10921
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034860.193, "dur": 1.617,
    "args": {
      "External id": 2489363,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10922
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034861.129, "dur": 0.613,
    "args": {
      "External id": 2489364,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10923
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034862.052, "dur": 2.281,
    "args": {
      "External id": 2489365,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10924
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034862.638, "dur": 1.626,
    "args": {
      "External id": 2489366,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10925
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034866.158, "dur": 1.472,
    "args": {
      "External id": 2489367,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10926
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034867.118, "dur": 0.443,
    "args": {
      "External id": 2489368,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10927
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034867.873, "dur": 1.427,
    "args": {
      "External id": 2489369,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10928
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034868.653, "dur": 0.585,
    "args": {
      "External id": 2489370,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10929
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034869.525, "dur": 3.026,
    "args": {
      "External id": 2489371,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10930
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034871.945, "dur": 0.538,
    "args": {
      "External id": 2489372,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10931
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034872.815, "dur": 1.283,
    "args": {
      "External id": 2489373,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10932
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034873.470, "dur": 0.560,
    "args": {
      "External id": 2489374,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10933
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034874.322, "dur": 1.394,
    "args": {
      "External id": 2489375,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10934
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034875.114, "dur": 0.537,
    "args": {
      "External id": 2489376,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10935
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034875.960, "dur": 2.209,
    "args": {
      "External id": 2489377,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10936
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034877.650, "dur": 0.456,
    "args": {
      "External id": 2489378,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10937
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034878.414, "dur": 1.703,
    "args": {
      "External id": 2489379,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10938
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034879.282, "dur": 0.565,
    "args": {
      "External id": 2489380,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10939
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034880.357, "dur": 2.233,
    "args": {
      "External id": 2489381,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10940
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034881.058, "dur": 1.464,
    "args": {
      "External id": 2489382,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10941
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034884.274, "dur": 1.572,
    "args": {
      "External id": 2489383,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10942
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034885.222, "dur": 0.558,
    "args": {
      "External id": 2489384,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10943
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034886.087, "dur": 1.356,
    "args": {
      "External id": 2489385,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10944
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034886.684, "dur": 0.694,
    "args": {
      "External id": 2489386,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10945
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034887.668, "dur": 3.119,
    "args": {
      "External id": 2489387,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10946
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034890.206, "dur": 0.512,
    "args": {
      "External id": 2489388,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10947
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034891.098, "dur": 1.612,
    "args": {
      "External id": 2489389,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10948
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034891.830, "dur": 0.813,
    "args": {
      "External id": 2489390,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10949
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034892.935, "dur": 1.546,
    "args": {
      "External id": 2489391,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10950
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034893.901, "dur": 0.513,
    "args": {
      "External id": 2489392,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10951
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034894.743, "dur": 2.643,
    "args": {
      "External id": 2489393,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10952
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034896.827, "dur": 0.488,
    "args": {
      "External id": 2489394,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10953
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034897.622, "dur": 1.628,
    "args": {
      "External id": 2489395,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10954
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034898.521, "dur": 0.662,
    "args": {
      "External id": 2489396,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10955
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034899.492, "dur": 2.558,
    "args": {
      "External id": 2489397,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10956
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034900.255, "dur": 1.722,
    "args": {
      "External id": 2489398,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10957
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034903.700, "dur": 1.338,
    "args": {
      "External id": 2489399,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10958
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034904.610, "dur": 0.363,
    "args": {
      "External id": 2489400,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10959
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034905.288, "dur": 1.166,
    "args": {
      "External id": 2489401,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10960
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034905.856, "dur": 0.535,
    "args": {
      "External id": 2489402,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10961
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034906.681, "dur": 3.335,
    "args": {
      "External id": 2489403,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10962
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034909.348, "dur": 0.602,
    "args": {
      "External id": 2489404,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10963
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034910.297, "dur": 1.373,
    "args": {
      "External id": 2489405,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10964
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034910.938, "dur": 0.667,
    "args": {
      "External id": 2489406,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10965
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034911.894, "dur": 1.602,
    "args": {
      "External id": 2489407,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10966
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034912.802, "dur": 0.623,
    "args": {
      "External id": 2489408,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10967
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034913.744, "dur": 2.716,
    "args": {
      "External id": 2489409,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10968
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034915.629, "dur": 0.765,
    "args": {
      "External id": 2489410,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10969
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034916.795, "dur": 1.702,
    "args": {
      "External id": 2489411,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10970
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034917.609, "dur": 0.820,
    "args": {
      "External id": 2489412,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10971
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034918.736, "dur": 1.980,
    "args": {
      "External id": 2489413,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10972
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034919.322, "dur": 1.329,
    "args": {
      "External id": 2489414,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10973
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034922.261, "dur": 2.021,
    "args": {
      "External id": 2489415,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10974
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034923.293, "dur": 0.835,
    "args": {
      "External id": 2489416,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10975
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034924.525, "dur": 1.370,
    "args": {
      "External id": 2489417,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10976
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034925.222, "dur": 0.612,
    "args": {
      "External id": 2489418,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10977
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034926.121, "dur": 2.706,
    "args": {
      "External id": 2489419,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10978
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034928.255, "dur": 0.501,
    "args": {
      "External id": 2489420,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10979
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034929.092, "dur": 1.252,
    "args": {
      "External id": 2489421,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10980
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034929.686, "dur": 0.497,
    "args": {
      "External id": 2489422,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10981
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034930.566, "dur": 1.656,
    "args": {
      "External id": 2489423,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10982
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034931.526, "dur": 0.625,
    "args": {
      "External id": 2489424,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10983
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034932.462, "dur": 2.396,
    "args": {
      "External id": 2489425,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10984
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034934.241, "dur": 0.549,
    "args": {
      "External id": 2489426,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10985
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034935.082, "dur": 2.031,
    "args": {
      "External id": 2489427,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10986
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034936.442, "dur": 0.511,
    "args": {
      "External id": 2489428,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10987
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034937.367, "dur": 2.522,
    "args": {
      "External id": 2489429,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10988
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034938.075, "dur": 1.747,
    "args": {
      "External id": 2489430,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10989
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034941.613, "dur": 1.524,
    "args": {
      "External id": 2489431,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10990
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034942.479, "dur": 0.591,
    "args": {
      "External id": 2489432,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10991
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034943.485, "dur": 1.239,
    "args": {
      "External id": 2489433,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10992
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034944.136, "dur": 0.525,
    "args": {
      "External id": 2489434,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10993
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034944.970, "dur": 2.710,
    "args": {
      "External id": 2489435,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10994
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034947.303, "dur": 0.315,
    "args": {
      "External id": 2489436,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10995
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034947.965, "dur": 1.229,
    "args": {
      "External id": 2489437,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10996
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034948.619, "dur": 0.409,
    "args": {
      "External id": 2489438,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10997
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034949.414, "dur": 1.869,
    "args": {
      "External id": 2489439,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10998
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034950.574, "dur": 0.646,
    "args": {
      "External id": 2489440,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10999
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034951.549, "dur": 2.493,
    "args": {
      "External id": 2489441,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11000
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034953.581, "dur": 0.396,
    "args": {
      "External id": 2489442,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11001
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034954.387, "dur": 1.475,
    "args": {
      "External id": 2489443,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11002
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034955.198, "dur": 0.508,
    "args": {
      "External id": 2489444,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11003
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034956.110, "dur": 2.457,
    "args": {
      "External id": 2489445,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11004
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034956.872, "dur": 1.627,
    "args": {
      "External id": 2489446,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11005
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034960.171, "dur": 1.431,
    "args": {
      "External id": 2489447,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11006
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034961.114, "dur": 0.422,
    "args": {
      "External id": 2489448,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11007
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034961.947, "dur": 1.242,
    "args": {
      "External id": 2489449,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11008
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034962.562, "dur": 0.552,
    "args": {
      "External id": 2489450,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11009
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034963.414, "dur": 2.899,
    "args": {
      "External id": 2489451,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11010
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034965.809, "dur": 0.436,
    "args": {
      "External id": 2489452,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11011
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034966.578, "dur": 1.376,
    "args": {
      "External id": 2489453,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11012
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034967.446, "dur": 0.446,
    "args": {
      "External id": 2489454,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11013
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034968.180, "dur": 1.478,
    "args": {
      "External id": 2489455,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11014
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034969.087, "dur": 0.506,
    "args": {
      "External id": 2489456,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11015
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034969.905, "dur": 2.662,
    "args": {
      "External id": 2489457,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11016
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034972.087, "dur": 0.418,
    "args": {
      "External id": 2489458,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11017
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034972.794, "dur": 1.288,
    "args": {
      "External id": 2489459,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11018
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034973.493, "dur": 0.518,
    "args": {
      "External id": 2489460,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11019
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034974.322, "dur": 2.516,
    "args": {
      "External id": 2489461,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11020
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034975.140, "dur": 1.633,
    "args": {
      "External id": 2489462,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11021
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034978.667, "dur": 1.548,
    "args": {
      "External id": 2489463,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11022
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034979.724, "dur": 0.425,
    "args": {
      "External id": 2489464,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11023
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034980.599, "dur": 27.495,
    "args": {
      "External id": 2489465,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11024
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658034981.466, "dur": 25.767,
    "args": {
      "External id": 2489466,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11025
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035009.722, "dur": 3.523,
    "args": {
      "External id": 2489467,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11026
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035012.544, "dur": 0.632,
    "args": {
      "External id": 2489468,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11027
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035013.540, "dur": 1.198,
    "args": {
      "External id": 2489469,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11028
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035014.165, "dur": 0.381,
    "args": {
      "External id": 2489470,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11029
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035015.002, "dur": 1.492,
    "args": {
      "External id": 2489471,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11030
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035015.857, "dur": 0.576,
    "args": {
      "External id": 2489472,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11031
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035016.759, "dur": 2.389,
    "args": {
      "External id": 2489473,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11032
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035018.666, "dur": 0.418,
    "args": {
      "External id": 2489474,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11033
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035019.512, "dur": 2.925,
    "args": {
      "External id": 2489475,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11034
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035021.940, "dur": 0.434,
    "args": {
      "External id": 2489476,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11035
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035022.683, "dur": 2.458,
    "args": {
      "External id": 2489477,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11036
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035023.475, "dur": 1.596,
    "args": {
      "External id": 2489478,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11037
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035027.108, "dur": 1.449,
    "args": {
      "External id": 2489479,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11038
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035027.811, "dur": 0.666,
    "args": {
      "External id": 2489480,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11039
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035028.916, "dur": 1.178,
    "args": {
      "External id": 2489481,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11040
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035029.523, "dur": 0.507,
    "args": {
      "External id": 2489482,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11041
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035030.336, "dur": 3.208,
    "args": {
      "External id": 2489483,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11042
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035032.953, "dur": 0.523,
    "args": {
      "External id": 2489484,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11043
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035033.833, "dur": 1.889,
    "args": {
      "External id": 2489485,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11044
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035034.634, "dur": 1.010,
    "args": {
      "External id": 2489486,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11045
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035036.133, "dur": 2.026,
    "args": {
      "External id": 2489487,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11046
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035037.240, "dur": 0.857,
    "args": {
      "External id": 2489488,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11047
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035038.402, "dur": 2.658,
    "args": {
      "External id": 2489489,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11048
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035040.609, "dur": 0.384,
    "args": {
      "External id": 2489490,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11049
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035041.381, "dur": 1.381,
    "args": {
      "External id": 2489491,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11050
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035042.254, "dur": 0.447,
    "args": {
      "External id": 2489492,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11051
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035043.039, "dur": 2.316,
    "args": {
      "External id": 2489493,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11052
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035043.770, "dur": 1.519,
    "args": {
      "External id": 2489494,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11053
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035047.156, "dur": 1.680,
    "args": {
      "External id": 2489495,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11054
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035048.283, "dur": 0.493,
    "args": {
      "External id": 2489496,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11055
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035049.080, "dur": 1.276,
    "args": {
      "External id": 2489497,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11056
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035049.683, "dur": 0.608,
    "args": {
      "External id": 2489498,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11057
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035050.580, "dur": 3.174,
    "args": {
      "External id": 2489499,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11058
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035053.089, "dur": 0.595,
    "args": {
      "External id": 2489500,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11059
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035054.073, "dur": 1.394,
    "args": {
      "External id": 2489501,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11060
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035054.642, "dur": 0.760,
    "args": {
      "External id": 2489502,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11061
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035055.909, "dur": 1.665,
    "args": {
      "External id": 2489503,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11062
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035056.874, "dur": 0.637,
    "args": {
      "External id": 2489504,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11063
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035057.957, "dur": 2.498,
    "args": {
      "External id": 2489505,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11064
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035059.949, "dur": 0.440,
    "args": {
      "External id": 2489506,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11065
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035060.764, "dur": 1.588,
    "args": {
      "External id": 2489507,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11066
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035061.815, "dur": 0.472,
    "args": {
      "External id": 2489508,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11067
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035062.622, "dur": 2.028,
    "args": {
      "External id": 2489509,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11068
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035063.188, "dur": 1.394,
    "args": {
      "External id": 2489510,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11069
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035066.175, "dur": 1.587,
    "args": {
      "External id": 2489511,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11070
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035067.161, "dur": 0.535,
    "args": {
      "External id": 2489512,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11071
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035068.162, "dur": 1.298,
    "args": {
      "External id": 2489513,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11072
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035068.855, "dur": 0.541,
    "args": {
      "External id": 2489514,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11073
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035069.857, "dur": 3.304,
    "args": {
      "External id": 2489515,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11074
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035072.486, "dur": 0.603,
    "args": {
      "External id": 2489516,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11075
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035073.422, "dur": 1.326,
    "args": {
      "External id": 2489517,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11076
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035074.133, "dur": 0.555,
    "args": {
      "External id": 2489518,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11077
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035074.995, "dur": 1.791,
    "args": {
      "External id": 2489519,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11078
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035076.043, "dur": 0.679,
    "args": {
      "External id": 2489520,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11079
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035077.148, "dur": 2.643,
    "args": {
      "External id": 2489521,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11080
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035078.968, "dur": 0.760,
    "args": {
      "External id": 2489522,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11081
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035080.015, "dur": 1.530,
    "args": {
      "External id": 2489523,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11082
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035081.021, "dur": 0.458,
    "args": {
      "External id": 2489524,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11083
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035081.881, "dur": 2.594,
    "args": {
      "External id": 2489525,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11084
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035082.589, "dur": 1.820,
    "args": {
      "External id": 2489526,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11085
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035085.887, "dur": 1.513,
    "args": {
      "External id": 2489527,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11086
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035086.840, "dur": 0.490,
    "args": {
      "External id": 2489528,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11087
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035087.880, "dur": 1.417,
    "args": {
      "External id": 2489529,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11088
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035088.645, "dur": 0.589,
    "args": {
      "External id": 2489530,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11089
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035089.609, "dur": 3.052,
    "args": {
      "External id": 2489531,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11090
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035092.133, "dur": 0.462,
    "args": {
      "External id": 2489532,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11091
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035092.924, "dur": 1.303,
    "args": {
      "External id": 2489533,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11092
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035093.655, "dur": 0.510,
    "args": {
      "External id": 2489534,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11093
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035094.450, "dur": 1.907,
    "args": {
      "External id": 2489535,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11094
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035095.420, "dur": 0.876,
    "args": {
      "External id": 2489536,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11095
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035096.791, "dur": 2.872,
    "args": {
      "External id": 2489537,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11096
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035099.047, "dur": 0.555,
    "args": {
      "External id": 2489538,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11097
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035099.895, "dur": 1.645,
    "args": {
      "External id": 2489539,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11098
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035100.776, "dur": 0.690,
    "args": {
      "External id": 2489540,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11099
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035101.910, "dur": 2.585,
    "args": {
      "External id": 2489541,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11100
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035102.607, "dur": 1.820,
    "args": {
      "External id": 2489542,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11101
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035106.473, "dur": 1.579,
    "args": {
      "External id": 2489543,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11102
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035107.517, "dur": 0.458,
    "args": {
      "External id": 2489544,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11103
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035108.291, "dur": 1.234,
    "args": {
      "External id": 2489545,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11104
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035108.883, "dur": 0.570,
    "args": {
      "External id": 2489546,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11105
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035109.748, "dur": 3.014,
    "args": {
      "External id": 2489547,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11106
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035112.207, "dur": 0.486,
    "args": {
      "External id": 2489548,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11107
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035113.034, "dur": 1.357,
    "args": {
      "External id": 2489549,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11108
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035113.638, "dur": 0.688,
    "args": {
      "External id": 2489550,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11109
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035114.613, "dur": 1.743,
    "args": {
      "External id": 2489551,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11110
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035115.759, "dur": 0.529,
    "args": {
      "External id": 2489552,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11111
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035116.693, "dur": 2.706,
    "args": {
      "External id": 2489553,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11112
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035118.799, "dur": 0.536,
    "args": {
      "External id": 2489554,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11113
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035119.663, "dur": 1.626,
    "args": {
      "External id": 2489555,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11114
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035120.701, "dur": 0.524,
    "args": {
      "External id": 2489556,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11115
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035121.539, "dur": 2.302,
    "args": {
      "External id": 2489557,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11116
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035122.233, "dur": 1.539,
    "args": {
      "External id": 2489558,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11117
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035125.952, "dur": 1.737,
    "args": {
      "External id": 2489559,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11118
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035126.980, "dur": 0.643,
    "args": {
      "External id": 2489560,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11119
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035128.055, "dur": 1.329,
    "args": {
      "External id": 2489561,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11120
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035128.768, "dur": 0.552,
    "args": {
      "External id": 2489562,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11121
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035129.650, "dur": 2.941,
    "args": {
      "External id": 2489563,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11122
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035132.101, "dur": 0.420,
    "args": {
      "External id": 2489564,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11123
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035133.065, "dur": 1.097,
    "args": {
      "External id": 2489565,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11124
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035133.651, "dur": 0.434,
    "args": {
      "External id": 2489566,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11125
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035134.388, "dur": 1.400,
    "args": {
      "External id": 2489567,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11126
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035135.112, "dur": 0.613,
    "args": {
      "External id": 2489568,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11127
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035136.062, "dur": 2.269,
    "args": {
      "External id": 2489569,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11128
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035137.850, "dur": 0.418,
    "args": {
      "External id": 2489570,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11129
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035138.552, "dur": 1.370,
    "args": {
      "External id": 2489571,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11130
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035139.417, "dur": 0.442,
    "args": {
      "External id": 2489572,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11131
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035140.161, "dur": 2.500,
    "args": {
      "External id": 2489573,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11132
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035140.799, "dur": 1.796,
    "args": {
      "External id": 2489574,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11133
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035144.472, "dur": 1.368,
    "args": {
      "External id": 2489575,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11134
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035145.331, "dur": 0.448,
    "args": {
      "External id": 2489576,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11135
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035146.194, "dur": 1.307,
    "args": {
      "External id": 2489577,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11136
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035146.836, "dur": 0.601,
    "args": {
      "External id": 2489578,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11137
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035147.805, "dur": 3.566,
    "args": {
      "External id": 2489579,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11138
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035150.494, "dur": 0.808,
    "args": {
      "External id": 2489580,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11139
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035151.631, "dur": 1.274,
    "args": {
      "External id": 2489581,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11140
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035152.378, "dur": 0.459,
    "args": {
      "External id": 2489582,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11141
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035153.152, "dur": 1.623,
    "args": {
      "External id": 2489583,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11142
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035153.871, "dur": 0.838,
    "args": {
      "External id": 2489584,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11143
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035155.046, "dur": 2.685,
    "args": {
      "External id": 2489585,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11144
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035157.087, "dur": 0.581,
    "args": {
      "External id": 2489586,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11145
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035158.071, "dur": 1.686,
    "args": {
      "External id": 2489587,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11146
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035159.115, "dur": 0.573,
    "args": {
      "External id": 2489588,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11147
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035160.021, "dur": 2.941,
    "args": {
      "External id": 2489589,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11148
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035160.612, "dur": 2.282,
    "args": {
      "External id": 2489590,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11149
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035164.547, "dur": 1.592,
    "args": {
      "External id": 2489591,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11150
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035165.441, "dur": 0.631,
    "args": {
      "External id": 2489592,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11151
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035166.376, "dur": 1.508,
    "args": {
      "External id": 2489593,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11152
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035167.080, "dur": 0.741,
    "args": {
      "External id": 2489594,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11153
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035168.105, "dur": 3.632,
    "args": {
      "External id": 2489595,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11154
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035171.031, "dur": 0.639,
    "args": {
      "External id": 2489596,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11155
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035172.001, "dur": 1.189,
    "args": {
      "External id": 2489597,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11156
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035172.605, "dur": 0.525,
    "args": {
      "External id": 2489598,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11157
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035173.436, "dur": 1.595,
    "args": {
      "External id": 2489599,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11158
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035174.322, "dur": 0.647,
    "args": {
      "External id": 2489600,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11159
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035175.275, "dur": 2.508,
    "args": {
      "External id": 2489601,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11160
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035177.030, "dur": 0.693,
    "args": {
      "External id": 2489602,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11161
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035178.005, "dur": 1.372,
    "args": {
      "External id": 2489603,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11162
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035178.821, "dur": 0.491,
    "args": {
      "External id": 2489604,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11163
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035179.616, "dur": 2.014,
    "args": {
      "External id": 2489605,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11164
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035180.192, "dur": 1.370,
    "args": {
      "External id": 2489606,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11165
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035183.192, "dur": 1.708,
    "args": {
      "External id": 2489607,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11166
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035183.988, "dur": 0.847,
    "args": {
      "External id": 2489608,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11167
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035185.157, "dur": 1.564,
    "args": {
      "External id": 2489609,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11168
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035185.746, "dur": 0.904,
    "args": {
      "External id": 2489610,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11169
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035186.976, "dur": 3.428,
    "args": {
      "External id": 2489611,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11170
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035189.563, "dur": 0.771,
    "args": {
      "External id": 2489612,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11171
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035190.673, "dur": 1.326,
    "args": {
      "External id": 2489613,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11172
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035191.268, "dur": 0.662,
    "args": {
      "External id": 2489614,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11173
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035192.239, "dur": 1.833,
    "args": {
      "External id": 2489615,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11174
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035193.381, "dur": 0.626,
    "args": {
      "External id": 2489616,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11175
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035194.310, "dur": 2.419,
    "args": {
      "External id": 2489617,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11176
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035196.225, "dur": 0.433,
    "args": {
      "External id": 2489618,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11177
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035196.973, "dur": 1.503,
    "args": {
      "External id": 2489619,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11178
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035197.936, "dur": 0.476,
    "args": {
      "External id": 2489620,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11179
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035198.738, "dur": 3.137,
    "args": {
      "External id": 2489621,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11180
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035199.655, "dur": 2.064,
    "args": {
      "External id": 2489622,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11181
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035203.385, "dur": 1.740,
    "args": {
      "External id": 2489623,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11182
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035204.246, "dur": 0.810,
    "args": {
      "External id": 2489624,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11183
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035205.364, "dur": 1.427,
    "args": {
      "External id": 2489625,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11184
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035205.921, "dur": 0.807,
    "args": {
      "External id": 2489626,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11185
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035207.100, "dur": 3.019,
    "args": {
      "External id": 2489627,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11186
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035209.339, "dur": 0.709,
    "args": {
      "External id": 2489628,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11187
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035210.381, "dur": 1.043,
    "args": {
      "External id": 2489629,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11188
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035210.938, "dur": 0.423,
    "args": {
      "External id": 2489630,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11189
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035211.661, "dur": 1.553,
    "args": {
      "External id": 2489631,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11190
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035212.603, "dur": 0.547,
    "args": {
      "External id": 2489632,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11191
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035213.479, "dur": 2.452,
    "args": {
      "External id": 2489633,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11192
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035215.440, "dur": 0.422,
    "args": {
      "External id": 2489634,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11193
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035216.152, "dur": 1.504,
    "args": {
      "External id": 2489635,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11194
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035217.070, "dur": 0.515,
    "args": {
      "External id": 2489636,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11195
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035217.896, "dur": 2.149,
    "args": {
      "External id": 2489637,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11196
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035218.463, "dur": 1.516,
    "args": {
      "External id": 2489638,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11197
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035221.784, "dur": 1.508,
    "args": {
      "External id": 2489639,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11198
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035222.639, "dur": 0.591,
    "args": {
      "External id": 2489640,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11199
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035223.630, "dur": 1.171,
    "args": {
      "External id": 2489641,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11200
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035224.201, "dur": 0.534,
    "args": {
      "External id": 2489642,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11201
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035225.166, "dur": 3.408,
    "args": {
      "External id": 2489643,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11202
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035228.116, "dur": 0.395,
    "args": {
      "External id": 2489644,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11203
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035228.841, "dur": 1.199,
    "args": {
      "External id": 2489645,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11204
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035229.440, "dur": 0.537,
    "args": {
      "External id": 2489646,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11205
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035230.310, "dur": 1.403,
    "args": {
      "External id": 2489647,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11206
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035231.076, "dur": 0.571,
    "args": {
      "External id": 2489648,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11207
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035231.963, "dur": 2.360,
    "args": {
      "External id": 2489649,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11208
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035233.920, "dur": 0.332,
    "args": {
      "External id": 2489650,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11209
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035234.545, "dur": 1.454,
    "args": {
      "External id": 2489651,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11210
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035235.517, "dur": 0.420,
    "args": {
      "External id": 2489652,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11211
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035236.259, "dur": 2.328,
    "args": {
      "External id": 2489653,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11212
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035236.838, "dur": 1.401,
    "args": {
      "External id": 2489654,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11213
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035240.059, "dur": 1.623,
    "args": {
      "External id": 2489655,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11214
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035241.120, "dur": 0.495,
    "args": {
      "External id": 2489656,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11215
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035241.920, "dur": 1.557,
    "args": {
      "External id": 2489657,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11216
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035242.536, "dur": 0.879,
    "args": {
      "External id": 2489658,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11217
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035243.911, "dur": 2.777,
    "args": {
      "External id": 2489659,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11218
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035246.193, "dur": 0.417,
    "args": {
      "External id": 2489660,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11219
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035246.984, "dur": 1.085,
    "args": {
      "External id": 2489661,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11220
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035247.566, "dur": 0.434,
    "args": {
      "External id": 2489662,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11221
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035248.290, "dur": 1.395,
    "args": {
      "External id": 2489663,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11222
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035249.078, "dur": 0.546,
    "args": {
      "External id": 2489664,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11223
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035250.227, "dur": 2.293,
    "args": {
      "External id": 2489665,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11224
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035252.018, "dur": 0.439,
    "args": {
      "External id": 2489666,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11225
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035252.832, "dur": 1.453,
    "args": {
      "External id": 2489667,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11226
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035253.742, "dur": 0.474,
    "args": {
      "External id": 2489668,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11227
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035254.546, "dur": 2.712,
    "args": {
      "External id": 2489669,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11228
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035255.209, "dur": 1.864,
    "args": {
      "External id": 2489670,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11229
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035259.129, "dur": 1.728,
    "args": {
      "External id": 2489671,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11230
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035260.164, "dur": 0.625,
    "args": {
      "External id": 2489672,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11231
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035261.199, "dur": 1.486,
    "args": {
      "External id": 2489673,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11232
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035261.765, "dur": 0.855,
    "args": {
      "External id": 2489674,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11233
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035262.908, "dur": 2.922,
    "args": {
      "External id": 2489675,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11234
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035265.307, "dur": 0.454,
    "args": {
      "External id": 2489676,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11235
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035266.116, "dur": 1.335,
    "args": {
      "External id": 2489677,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11236
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035266.792, "dur": 0.591,
    "args": {
      "External id": 2489678,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11237
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035267.685, "dur": 1.902,
    "args": {
      "External id": 2489679,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11238
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035268.532, "dur": 0.984,
    "args": {
      "External id": 2489680,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11239
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035270.188, "dur": 2.287,
    "args": {
      "External id": 2489681,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11240
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035271.987, "dur": 0.425,
    "args": {
      "External id": 2489682,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11241
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035272.794, "dur": 1.572,
    "args": {
      "External id": 2489683,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11242
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035273.832, "dur": 0.473,
    "args": {
      "External id": 2489684,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11243
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035274.635, "dur": 2.308,
    "args": {
      "External id": 2489685,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11244
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035275.371, "dur": 1.505,
    "args": {
      "External id": 2489686,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11245
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035278.492, "dur": 1.430,
    "args": {
      "External id": 2489687,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11246
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035279.354, "dur": 0.503,
    "args": {
      "External id": 2489688,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11247
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035280.354, "dur": 1.216,
    "args": {
      "External id": 2489689,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11248
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035280.954, "dur": 0.552,
    "args": {
      "External id": 2489690,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11249
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035281.818, "dur": 3.133,
    "args": {
      "External id": 2489691,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11250
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035284.360, "dur": 0.525,
    "args": {
      "External id": 2489692,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11251
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035285.213, "dur": 1.324,
    "args": {
      "External id": 2489693,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11252
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035285.933, "dur": 0.538,
    "args": {
      "External id": 2489694,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11253
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035286.766, "dur": 1.697,
    "args": {
      "External id": 2489695,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11254
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035287.740, "dur": 0.658,
    "args": {
      "External id": 2489696,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11255
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035288.808, "dur": 2.333,
    "args": {
      "External id": 2489697,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11256
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035290.518, "dur": 0.546,
    "args": {
      "External id": 2489698,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11257
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035291.383, "dur": 1.370,
    "args": {
      "External id": 2489699,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11258
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035292.238, "dur": 0.447,
    "args": {
      "External id": 2489700,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11259
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035293.005, "dur": 2.262,
    "args": {
      "External id": 2489701,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11260
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035293.747, "dur": 1.455,
    "args": {
      "External id": 2489702,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11261
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035297.080, "dur": 1.597,
    "args": {
      "External id": 2489703,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11262
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035297.895, "dur": 0.716,
    "args": {
      "External id": 2489704,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11263
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035298.920, "dur": 1.259,
    "args": {
      "External id": 2489705,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11264
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035299.537, "dur": 0.576,
    "args": {
      "External id": 2489706,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11265
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035300.479, "dur": 3.170,
    "args": {
      "External id": 2489707,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11266
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035303.068, "dur": 0.516,
    "args": {
      "External id": 2489708,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11267
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035303.940, "dur": 1.233,
    "args": {
      "External id": 2489709,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11268
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035304.679, "dur": 0.431,
    "args": {
      "External id": 2489710,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11269
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035305.397, "dur": 1.597,
    "args": {
      "External id": 2489711,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11270
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035306.358, "dur": 0.571,
    "args": {
      "External id": 2489712,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11271
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035307.340, "dur": 2.356,
    "args": {
      "External id": 2489713,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11272
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035309.211, "dur": 0.420,
    "args": {
      "External id": 2489714,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11273
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035310.144, "dur": 1.419,
    "args": {
      "External id": 2489715,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11274
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035311.032, "dur": 0.464,
    "args": {
      "External id": 2489716,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11275
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035311.958, "dur": 2.252,
    "args": {
      "External id": 2489717,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11276
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035312.660, "dur": 1.470,
    "args": {
      "External id": 2489718,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11277
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035316.004, "dur": 1.395,
    "args": {
      "External id": 2489719,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11278
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035316.896, "dur": 0.441,
    "args": {
      "External id": 2489720,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11279
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035317.936, "dur": 1.245,
    "args": {
      "External id": 2489721,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11280
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035318.565, "dur": 0.551,
    "args": {
      "External id": 2489722,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11281
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035319.419, "dur": 2.492,
    "args": {
      "External id": 2489723,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11282
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035321.414, "dur": 0.426,
    "args": {
      "External id": 2489724,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11283
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035322.173, "dur": 1.114,
    "args": {
      "External id": 2489725,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11284
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035322.747, "dur": 0.476,
    "args": {
      "External id": 2489726,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11285
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035323.536, "dur": 1.691,
    "args": {
      "External id": 2489727,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11286
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035324.568, "dur": 0.595,
    "args": {
      "External id": 2489728,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11287
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035325.600, "dur": 2.962,
    "args": {
      "External id": 2489729,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11288
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035327.862, "dur": 0.632,
    "args": {
      "External id": 2489730,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11289
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035328.782, "dur": 2.745,
    "args": {
      "External id": 2489731,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11290
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035331.048, "dur": 0.412,
    "args": {
      "External id": 2489732,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11291
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::cat", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035351.003, "dur": 133.918,
    "args": {
      "External id": 2489733,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["TensorList", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11292
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linalg_vector_norm", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035601.980, "dur": 135.615,
    "args": {
      "External id": 2489734,"Record function id": 0, "Concrete Inputs": ["", "2.", "", "False", ""], "Input type": ["float", "Scalar", "", "Scalar", ""], "Input Strides": [[1], [], [], [], []], "Input Dims": [[291], [], [], [], []], "Ev Idx": 11293
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linalg_vector_norm", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035665.865, "dur": 51.940,
    "args": {
      "External id": 2489735,"Record function id": 0, "Concrete Inputs": ["", "2.", "", "False", ""], "Input type": ["float", "Scalar", "", "Scalar", ""], "Input Strides": [[1], [], [], [], []], "Input Dims": [[291], [], [], [], []], "Ev Idx": 11294
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336754, "tid": 1336754,
    "ts": 1514658035681.298, "dur": 1.321,
    "args": {
      "External id": 2489736,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11295
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Redistribute", "pid": 1336754, "tid": 1336754,
    "ts": 1514658036176.547, "dur": 987.612,
    "args": {
      "External id": 2489737,"Sequence number": 24740656, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "False"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11296
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::pow", "pid": 1336754, "tid": 1336754,
    "ts": 1514658036232.548, "dur": 106.870,
    "args": {
      "External id": 2489738,"Record function id": 0, "Concrete Inputs": ["", "2."], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11297
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658036239.141, "dur": 1.457,
    "args": {
      "External id": 2489739,"Record function id": 0, "Concrete Inputs": ["", "2."], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11298
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658036291.981, "dur": 0.697,
    "args": {
      "External id": 2489740,"Record function id": 0, "Concrete Inputs": ["", "6", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 11299
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "_c10d_functional::all_reduce", "pid": 1336754, "tid": 1336754,
    "ts": 1514658036366.354, "dur": 449.479,
    "args": {
      "External id": 2489741,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["float", "", ""], "Input Strides": [[], [], []], "Input Dims": [[], [], []], "Ev Idx": 11300
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336754, "tid": 1336754,
    "ts": 1514658036372.078, "dur": 45.893,
    "args": {
      "External id": 2489742,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11301
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336754, "tid": 1336754,
    "ts": 1514658036375.548, "dur": 8.319,
    "args": {
      "External id": 2489743,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "0"], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 11302
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514658036379.357, "dur": 3.691,
    "args": {
      "External id": 2489744,"Record function id": 0, "Concrete Inputs": ["[]", "6", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 11303
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336754, "tid": 1336754,
    "ts": 1514658036385.517, "dur": 31.865,
    "args": {
      "External id": 2489745,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[], [], []], "Input Dims": [[], [], []], "Ev Idx": 11304
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::allreduce_", "pid": 1336754, "tid": 1336754,
    "ts": 1514658036427.651, "dur": 384.702,
    "args": {
      "External id": 2489746,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "-1"], "Input type": ["TensorList", "", "", "", "Scalar"], "Input Strides": [[[]], [], [], [], []], "Input Dims": [[[]], [], [], [], []], "Ev Idx": 11305
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514658036458.949, "dur": 346.257,
    "args": {
      "External id": 2489747,"Record function id": 0, "Collective name": "allreduce", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 1, "Input Strides": [[[]], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "1", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 1, "Process Group Name": "0", "Input type": ["TensorList", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[[]], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 11306, "In msg nelems": 1
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:all_reduce", "pid": 1336754, "tid": 1336754,
    "ts": 1514658036477.218, "dur": 321.677,
    "args": {
      "External id": 2489748,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 11307
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::pow", "pid": 1336754, "tid": 1336754,
    "ts": 1514658036879.710, "dur": 246.570,
    "args": {
      "External id": 2489749,"Record function id": 0, "Concrete Inputs": ["", "0.5"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11308
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "_c10d_functional::wait_tensor", "pid": 1336754, "tid": 1336754,
    "ts": 1514658036972.518, "dur": 66.787,
    "args": {
      "External id": 2489750,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 11309
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336754, "tid": 1336754,
    "ts": 1514658037023.007, "dur": 5.885,
    "args": {
      "External id": 2489751,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "1", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 11310, "In msg nelems": 0, "Rank": 1, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::pow", "pid": 1336754, "tid": 1336754,
    "ts": 1514658037074.645, "dur": 45.479,
    "args": {
      "External id": 2489752,"Record function id": 0, "Concrete Inputs": ["", "0.5"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11311
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658037077.259, "dur": 0.928,
    "args": {
      "External id": 2489753,"Record function id": 0, "Concrete Inputs": ["", "0.5"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11312
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658037079.629, "dur": 0.521,
    "args": {
      "External id": 2489754,"Record function id": 0, "Concrete Inputs": ["", "6", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 11313
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "_ToTorchTensor", "pid": 1336754, "tid": 1336754,
    "ts": 1514658037181.236, "dur": 22.150,
    "args": {
      "External id": 2489755,"Sequence number": 24740657, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", ""], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11314
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336754, "tid": 1336754,
    "ts": 1514658037191.979, "dur": 8.332,
    "args": {
      "External id": 2489756,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11315
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336754, "tid": 1336754,
    "ts": 1514658037194.444, "dur": 5.702,
    "args": {
      "External id": 2489757,"Record function id": 0, "Concrete Inputs": ["", "[]"], "Input type": ["float", "ScalarList"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11316
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336754, "tid": 1336754,
    "ts": 1514658037595.263, "dur": 40.046,
    "args": {
      "External id": 2489758,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "double", "Scalar"], "Input Strides": [[], [], []], "Input Dims": [[], [], []], "Ev Idx": 11317
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reciprocal", "pid": 1336754, "tid": 1336754,
    "ts": 1514658037645.352, "dur": 20.126,
    "args": {
      "External id": 2489759,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 11318
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mul", "pid": 1336754, "tid": 1336754,
    "ts": 1514658037672.416, "dur": 29.359,
    "args": {
      "External id": 2489760,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "double"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11319
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clamp", "pid": 1336754, "tid": 1336754,
    "ts": 1514658037719.311, "dur": 23.538,
    "args": {
      "External id": 2489761,"Record function id": 0, "Concrete Inputs": ["", "", "1."], "Input type": ["float", "", "Scalar"], "Input Strides": [[], [], []], "Input Dims": [[], [], []], "Ev Idx": 11320
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658037722.469, "dur": 0.403,
    "args": {
      "External id": 2489762,"Record function id": 0, "Concrete Inputs": ["", "6", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 11321
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336754, "tid": 1336754,
    "ts": 1514658037761.249, "dur": 0.470,
    "args": {
      "External id": 2489763,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 11322
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_mul_", "pid": 1336754, "tid": 1336754,
    "ts": 1514658037880.782, "dur": 773.788,
    "args": {
      "External id": 2489764,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["TensorList", "float"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11323
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_mul_", "pid": 1336754, "tid": 1336754,
    "ts": 1514658038422.242, "dur": 198.562,
    "args": {
      "External id": 2489765,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["TensorList", "float"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11324
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::isnan", "pid": 1336754, "tid": 1336754,
    "ts": 1514658038702.253, "dur": 38.006,
    "args": {
      "External id": 2489766,"Sequence number": 24740658, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 11325
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::ne", "pid": 1336754, "tid": 1336754,
    "ts": 1514658038706.202, "dur": 33.621,
    "args": {
      "External id": 2489767,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11326
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::is_nonzero", "pid": 1336754, "tid": 1336754,
    "ts": 1514658038745.148, "dur": 366.899,
    "args": {
      "External id": 2489768,"Sequence number": 24740658, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 11327
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::item", "pid": 1336754, "tid": 1336754,
    "ts": 1514658038747.116, "dur": 364.705,
    "args": {
      "External id": 2489769,"Sequence number": 24740658, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 11328
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_local_scalar_dense", "pid": 1336754, "tid": 1336754,
    "ts": 1514658038749.109, "dur": 361.850,
    "args": {
      "External id": 2489770,"Sequence number": 24740658, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 11329
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::isinf", "pid": 1336754, "tid": 1336754,
    "ts": 1514658039119.331, "dur": 62.618,
    "args": {
      "External id": 2489771,"Sequence number": 24740658, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 11330
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514658039123.159, "dur": 35.292,
    "args": {
      "External id": 2489772,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 11331
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336754, "tid": 1336754,
    "ts": 1514658039130.356, "dur": 3.333,
    "args": {
      "External id": 2489773,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 11332
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336754, "tid": 1336754,
    "ts": 1514658039135.607, "dur": 22.518,
    "args": {
      "External id": 2489774,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[], [1]], "Input Dims": [[], [0]], "Ev Idx": 11333
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336754, "tid": 1336754,
    "ts": 1514658039140.958, "dur": 2.731,
    "args": {
      "External id": 2489775,"Record function id": 0, "Concrete Inputs": ["", "[]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 11334
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336754, "tid": 1336754,
    "ts": 1514658039160.096, "dur": 20.930,
    "args": {
      "External id": 2489776,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11335
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::is_nonzero", "pid": 1336754, "tid": 1336754,
    "ts": 1514658039184.125, "dur": 38.010,
    "args": {
      "External id": 2489777,"Sequence number": 24740658, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 11336
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::item", "pid": 1336754, "tid": 1336754,
    "ts": 1514658039186.567, "dur": 35.428,
    "args": {
      "External id": 2489778,"Sequence number": 24740658, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 11337
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_local_scalar_dense", "pid": 1336754, "tid": 1336754,
    "ts": 1514658039187.408, "dur": 34.301,
    "args": {
      "External id": 2489779,"Sequence number": 24740658, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 11338
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "Optimizer.step#OptimizersContainer.step", "pid": 1336754, "tid": 1336754,
    "ts": 1514658039262.472, "dur": 5564.987,
    "args": {
      "External id": 2489780,"Record function id": 0, "Ev Idx": 11339
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "Optimizer.step#AdamW.step", "pid": 1336754, "tid": 1336754,
    "ts": 1514658039295.227, "dur": 5512.828,
    "args": {
      "External id": 2489781,"Record function id": 0, "Ev Idx": 11340
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_add_", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040514.554, "dur": 290.233,
    "args": {
      "External id": 2489782,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["TensorList", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11341
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040547.238, "dur": 1.056,
    "args": {
      "External id": 2489783,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11342
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040549.566, "dur": 0.329,
    "args": {
      "External id": 2489784,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11343
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040550.526, "dur": 0.207,
    "args": {
      "External id": 2489785,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11344
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040551.259, "dur": 0.132,
    "args": {
      "External id": 2489786,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11345
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040551.923, "dur": 0.172,
    "args": {
      "External id": 2489787,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11346
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040552.561, "dur": 0.075,
    "args": {
      "External id": 2489788,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11347
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040553.180, "dur": 0.185,
    "args": {
      "External id": 2489789,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11348
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040553.715, "dur": 0.168,
    "args": {
      "External id": 2489790,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11349
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040554.467, "dur": 0.254,
    "args": {
      "External id": 2489791,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11350
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040555.159, "dur": 0.219,
    "args": {
      "External id": 2489792,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11351
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040555.768, "dur": 0.100,
    "args": {
      "External id": 2489793,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11352
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040556.478, "dur": 0.050,
    "args": {
      "External id": 2489794,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11353
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040556.861, "dur": 0.059,
    "args": {
      "External id": 2489795,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11354
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040557.189, "dur": 0.062,
    "args": {
      "External id": 2489796,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11355
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040557.640, "dur": 0.073,
    "args": {
      "External id": 2489797,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11356
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040558.048, "dur": 0.064,
    "args": {
      "External id": 2489798,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11357
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040558.482, "dur": 0.058,
    "args": {
      "External id": 2489799,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11358
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040559.022, "dur": 0.061,
    "args": {
      "External id": 2489800,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11359
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040559.529, "dur": 0.069,
    "args": {
      "External id": 2489801,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11360
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040559.951, "dur": 0.059,
    "args": {
      "External id": 2489802,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11361
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040560.393, "dur": 0.067,
    "args": {
      "External id": 2489803,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11362
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040560.880, "dur": 0.053,
    "args": {
      "External id": 2489804,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11363
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040561.402, "dur": 0.060,
    "args": {
      "External id": 2489805,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11364
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040561.866, "dur": 0.055,
    "args": {
      "External id": 2489806,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11365
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040562.330, "dur": 0.062,
    "args": {
      "External id": 2489807,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11366
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040562.833, "dur": 0.229,
    "args": {
      "External id": 2489808,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11367
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040563.453, "dur": 0.183,
    "args": {
      "External id": 2489809,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11368
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040564.182, "dur": 0.073,
    "args": {
      "External id": 2489810,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11369
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040564.597, "dur": 0.065,
    "args": {
      "External id": 2489811,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11370
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040565.162, "dur": 0.072,
    "args": {
      "External id": 2489812,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11371
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040565.600, "dur": 0.066,
    "args": {
      "External id": 2489813,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11372
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040566.122, "dur": 0.065,
    "args": {
      "External id": 2489814,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11373
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040566.524, "dur": 0.182,
    "args": {
      "External id": 2489815,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11374
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040567.182, "dur": 0.168,
    "args": {
      "External id": 2489816,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11375
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040567.736, "dur": 0.089,
    "args": {
      "External id": 2489817,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11376
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040568.219, "dur": 0.068,
    "args": {
      "External id": 2489818,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11377
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040568.678, "dur": 0.069,
    "args": {
      "External id": 2489819,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11378
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040569.181, "dur": 0.051,
    "args": {
      "External id": 2489820,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11379
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040569.519, "dur": 0.059,
    "args": {
      "External id": 2489821,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11380
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040569.913, "dur": 0.069,
    "args": {
      "External id": 2489822,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11381
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040570.321, "dur": 0.064,
    "args": {
      "External id": 2489823,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11382
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040570.704, "dur": 0.072,
    "args": {
      "External id": 2489824,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11383
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040571.162, "dur": 0.071,
    "args": {
      "External id": 2489825,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11384
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040571.689, "dur": 0.067,
    "args": {
      "External id": 2489826,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11385
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040572.112, "dur": 0.060,
    "args": {
      "External id": 2489827,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11386
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040572.615, "dur": 0.056,
    "args": {
      "External id": 2489828,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11387
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040573.034, "dur": 0.059,
    "args": {
      "External id": 2489829,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11388
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040573.538, "dur": 0.057,
    "args": {
      "External id": 2489830,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11389
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040573.942, "dur": 0.056,
    "args": {
      "External id": 2489831,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11390
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040574.478, "dur": 0.056,
    "args": {
      "External id": 2489832,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11391
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040574.854, "dur": 0.057,
    "args": {
      "External id": 2489833,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11392
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040575.215, "dur": 0.058,
    "args": {
      "External id": 2489834,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11393
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040575.745, "dur": 0.055,
    "args": {
      "External id": 2489835,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11394
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040576.164, "dur": 0.057,
    "args": {
      "External id": 2489836,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11395
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040576.730, "dur": 0.045,
    "args": {
      "External id": 2489837,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11396
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040577.226, "dur": 0.051,
    "args": {
      "External id": 2489838,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11397
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040577.685, "dur": 0.067,
    "args": {
      "External id": 2489839,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11398
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040578.191, "dur": 0.180,
    "args": {
      "External id": 2489840,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11399
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040578.723, "dur": 0.226,
    "args": {
      "External id": 2489841,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11400
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040579.291, "dur": 0.221,
    "args": {
      "External id": 2489842,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11401
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040579.953, "dur": 0.058,
    "args": {
      "External id": 2489843,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11402
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040580.451, "dur": 0.151,
    "args": {
      "External id": 2489844,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11403
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040581.084, "dur": 0.168,
    "args": {
      "External id": 2489845,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11404
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040581.646, "dur": 0.232,
    "args": {
      "External id": 2489846,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11405
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040582.243, "dur": 0.182,
    "args": {
      "External id": 2489847,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11406
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040582.804, "dur": 0.201,
    "args": {
      "External id": 2489848,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11407
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040583.397, "dur": 0.083,
    "args": {
      "External id": 2489849,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11408
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040583.947, "dur": 0.221,
    "args": {
      "External id": 2489850,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11409
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040584.529, "dur": 0.067,
    "args": {
      "External id": 2489851,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11410
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040585.046, "dur": 0.072,
    "args": {
      "External id": 2489852,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11411
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040585.455, "dur": 0.196,
    "args": {
      "External id": 2489853,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11412
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040585.950, "dur": 0.187,
    "args": {
      "External id": 2489854,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11413
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040586.567, "dur": 0.220,
    "args": {
      "External id": 2489855,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11414
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040587.213, "dur": 0.067,
    "args": {
      "External id": 2489856,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11415
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040611.029, "dur": 0.154,
    "args": {
      "External id": 2489857,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11416
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040611.916, "dur": 0.052,
    "args": {
      "External id": 2489858,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11417
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040612.342, "dur": 0.068,
    "args": {
      "External id": 2489859,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11418
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040612.763, "dur": 0.064,
    "args": {
      "External id": 2489860,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11419
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040613.253, "dur": 0.056,
    "args": {
      "External id": 2489861,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11420
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040613.684, "dur": 0.069,
    "args": {
      "External id": 2489862,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11421
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040614.073, "dur": 0.067,
    "args": {
      "External id": 2489863,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11422
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040614.614, "dur": 0.051,
    "args": {
      "External id": 2489864,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11423
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040615.002, "dur": 0.064,
    "args": {
      "External id": 2489865,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11424
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040615.510, "dur": 0.060,
    "args": {
      "External id": 2489866,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11425
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040615.912, "dur": 0.059,
    "args": {
      "External id": 2489867,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11426
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040616.446, "dur": 0.067,
    "args": {
      "External id": 2489868,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11427
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040616.958, "dur": 0.061,
    "args": {
      "External id": 2489869,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11428
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040617.350, "dur": 0.070,
    "args": {
      "External id": 2489870,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11429
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040617.936, "dur": 0.068,
    "args": {
      "External id": 2489871,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11430
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040618.395, "dur": 0.065,
    "args": {
      "External id": 2489872,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11431
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040618.788, "dur": 0.056,
    "args": {
      "External id": 2489873,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11432
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040619.150, "dur": 0.063,
    "args": {
      "External id": 2489874,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11433
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040619.782, "dur": 0.062,
    "args": {
      "External id": 2489875,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11434
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040620.311, "dur": 0.071,
    "args": {
      "External id": 2489876,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11435
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040620.782, "dur": 0.071,
    "args": {
      "External id": 2489877,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11436
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040621.189, "dur": 0.062,
    "args": {
      "External id": 2489878,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11437
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040621.980, "dur": 0.066,
    "args": {
      "External id": 2489879,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11438
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040622.363, "dur": 0.065,
    "args": {
      "External id": 2489880,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11439
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040622.886, "dur": 0.067,
    "args": {
      "External id": 2489881,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11440
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040623.310, "dur": 0.066,
    "args": {
      "External id": 2489882,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11441
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040623.749, "dur": 0.057,
    "args": {
      "External id": 2489883,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11442
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040624.103, "dur": 0.067,
    "args": {
      "External id": 2489884,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11443
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040624.564, "dur": 0.064,
    "args": {
      "External id": 2489885,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11444
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040624.960, "dur": 0.060,
    "args": {
      "External id": 2489886,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11445
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040625.388, "dur": 0.069,
    "args": {
      "External id": 2489887,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11446
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040625.807, "dur": 0.072,
    "args": {
      "External id": 2489888,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11447
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040626.251, "dur": 0.059,
    "args": {
      "External id": 2489889,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11448
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040626.870, "dur": 0.060,
    "args": {
      "External id": 2489890,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11449
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040627.248, "dur": 0.064,
    "args": {
      "External id": 2489891,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11450
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040627.635, "dur": 0.066,
    "args": {
      "External id": 2489892,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11451
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040628.196, "dur": 0.063,
    "args": {
      "External id": 2489893,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11452
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040628.680, "dur": 0.056,
    "args": {
      "External id": 2489894,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11453
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040629.188, "dur": 0.071,
    "args": {
      "External id": 2489895,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11454
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040629.613, "dur": 0.069,
    "args": {
      "External id": 2489896,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11455
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040630.020, "dur": 0.056,
    "args": {
      "External id": 2489897,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11456
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040630.573, "dur": 0.063,
    "args": {
      "External id": 2489898,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11457
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040630.982, "dur": 0.058,
    "args": {
      "External id": 2489899,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11458
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040631.580, "dur": 0.066,
    "args": {
      "External id": 2489900,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11459
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040632.075, "dur": 0.069,
    "args": {
      "External id": 2489901,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11460
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040632.693, "dur": 0.066,
    "args": {
      "External id": 2489902,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11461
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040633.186, "dur": 0.067,
    "args": {
      "External id": 2489903,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11462
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040633.908, "dur": 0.052,
    "args": {
      "External id": 2489904,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11463
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040634.404, "dur": 0.058,
    "args": {
      "External id": 2489905,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11464
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040634.844, "dur": 0.066,
    "args": {
      "External id": 2489906,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11465
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040635.322, "dur": 0.062,
    "args": {
      "External id": 2489907,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11466
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040635.707, "dur": 0.054,
    "args": {
      "External id": 2489908,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11467
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040636.150, "dur": 0.064,
    "args": {
      "External id": 2489909,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11468
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040643.508, "dur": 0.082,
    "args": {
      "External id": 2489910,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11469
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040644.362, "dur": 0.054,
    "args": {
      "External id": 2489911,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11470
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040644.870, "dur": 0.067,
    "args": {
      "External id": 2489912,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11471
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040645.315, "dur": 0.063,
    "args": {
      "External id": 2489913,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11472
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040645.861, "dur": 0.067,
    "args": {
      "External id": 2489914,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11473
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040646.296, "dur": 0.065,
    "args": {
      "External id": 2489915,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11474
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040646.799, "dur": 0.067,
    "args": {
      "External id": 2489916,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11475
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040647.256, "dur": 0.068,
    "args": {
      "External id": 2489917,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11476
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040647.772, "dur": 0.058,
    "args": {
      "External id": 2489918,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11477
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040648.264, "dur": 0.069,
    "args": {
      "External id": 2489919,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11478
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040648.681, "dur": 0.072,
    "args": {
      "External id": 2489920,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11479
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040649.175, "dur": 0.058,
    "args": {
      "External id": 2489921,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11480
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040649.606, "dur": 0.058,
    "args": {
      "External id": 2489922,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11481
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040649.964, "dur": 0.067,
    "args": {
      "External id": 2489923,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11482
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040650.486, "dur": 0.065,
    "args": {
      "External id": 2489924,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11483
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040650.947, "dur": 0.066,
    "args": {
      "External id": 2489925,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11484
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040651.365, "dur": 0.070,
    "args": {
      "External id": 2489926,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11485
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040651.966, "dur": 0.054,
    "args": {
      "External id": 2489927,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11486
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040652.354, "dur": 0.069,
    "args": {
      "External id": 2489928,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11487
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040652.798, "dur": 0.067,
    "args": {
      "External id": 2489929,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11488
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040653.414, "dur": 0.064,
    "args": {
      "External id": 2489930,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11489
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040653.787, "dur": 0.059,
    "args": {
      "External id": 2489931,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11490
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040654.163, "dur": 0.065,
    "args": {
      "External id": 2489932,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11491
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040654.540, "dur": 0.069,
    "args": {
      "External id": 2489933,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11492
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040655.148, "dur": 0.062,
    "args": {
      "External id": 2489934,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11493
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040655.740, "dur": 0.068,
    "args": {
      "External id": 2489935,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11494
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040656.274, "dur": 0.074,
    "args": {
      "External id": 2489936,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11495
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040656.719, "dur": 0.056,
    "args": {
      "External id": 2489937,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11496
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040657.313, "dur": 0.069,
    "args": {
      "External id": 2489938,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11497
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040657.780, "dur": 0.060,
    "args": {
      "External id": 2489939,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11498
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040658.194, "dur": 0.060,
    "args": {
      "External id": 2489940,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11499
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040658.721, "dur": 0.059,
    "args": {
      "External id": 2489941,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11500
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040659.068, "dur": 0.071,
    "args": {
      "External id": 2489942,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11501
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040659.559, "dur": 0.056,
    "args": {
      "External id": 2489943,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11502
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040660.102, "dur": 0.058,
    "args": {
      "External id": 2489944,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11503
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040660.637, "dur": 0.056,
    "args": {
      "External id": 2489945,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11504
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040661.073, "dur": 0.067,
    "args": {
      "External id": 2489946,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11505
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040661.482, "dur": 0.059,
    "args": {
      "External id": 2489947,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11506
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040661.884, "dur": 0.067,
    "args": {
      "External id": 2489948,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11507
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040662.324, "dur": 0.069,
    "args": {
      "External id": 2489949,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11508
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040662.834, "dur": 0.070,
    "args": {
      "External id": 2489950,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11509
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040663.327, "dur": 0.072,
    "args": {
      "External id": 2489951,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11510
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040663.720, "dur": 0.072,
    "args": {
      "External id": 2489952,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11511
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040664.238, "dur": 0.059,
    "args": {
      "External id": 2489953,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11512
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040664.635, "dur": 0.060,
    "args": {
      "External id": 2489954,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11513
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040665.147, "dur": 0.054,
    "args": {
      "External id": 2489955,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11514
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040665.630, "dur": 0.067,
    "args": {
      "External id": 2489956,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11515
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040666.162, "dur": 0.055,
    "args": {
      "External id": 2489957,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11516
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040666.710, "dur": 0.061,
    "args": {
      "External id": 2489958,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11517
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040667.206, "dur": 0.057,
    "args": {
      "External id": 2489959,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11518
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040667.566, "dur": 0.058,
    "args": {
      "External id": 2489960,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11519
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040667.946, "dur": 0.068,
    "args": {
      "External id": 2489961,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11520
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040668.402, "dur": 0.068,
    "args": {
      "External id": 2489962,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11521
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040668.746, "dur": 0.067,
    "args": {
      "External id": 2489963,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11522
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040669.267, "dur": 0.067,
    "args": {
      "External id": 2489964,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11523
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040669.680, "dur": 0.068,
    "args": {
      "External id": 2489965,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11524
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040670.103, "dur": 0.070,
    "args": {
      "External id": 2489966,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11525
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040670.607, "dur": 0.071,
    "args": {
      "External id": 2489967,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11526
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040671.013, "dur": 0.073,
    "args": {
      "External id": 2489968,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11527
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040671.445, "dur": 0.057,
    "args": {
      "External id": 2489969,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11528
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040671.771, "dur": 0.070,
    "args": {
      "External id": 2489970,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11529
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040672.208, "dur": 0.065,
    "args": {
      "External id": 2489971,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11530
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040672.720, "dur": 0.066,
    "args": {
      "External id": 2489972,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11531
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040673.304, "dur": 0.066,
    "args": {
      "External id": 2489973,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11532
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040673.947, "dur": 0.050,
    "args": {
      "External id": 2489974,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11533
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040674.313, "dur": 0.071,
    "args": {
      "External id": 2489975,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11534
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040674.829, "dur": 0.070,
    "args": {
      "External id": 2489976,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11535
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040675.307, "dur": 0.060,
    "args": {
      "External id": 2489977,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11536
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040675.823, "dur": 0.070,
    "args": {
      "External id": 2489978,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11537
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040676.315, "dur": 0.069,
    "args": {
      "External id": 2489979,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11538
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040676.814, "dur": 0.054,
    "args": {
      "External id": 2489980,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11539
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040677.257, "dur": 0.059,
    "args": {
      "External id": 2489981,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11540
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040678.080, "dur": 0.058,
    "args": {
      "External id": 2489982,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11541
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040678.501, "dur": 0.059,
    "args": {
      "External id": 2489983,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11542
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040678.860, "dur": 0.058,
    "args": {
      "External id": 2489984,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11543
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040679.352, "dur": 0.054,
    "args": {
      "External id": 2489985,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11544
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040679.780, "dur": 0.057,
    "args": {
      "External id": 2489986,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11545
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040680.201, "dur": 0.059,
    "args": {
      "External id": 2489987,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11546
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040680.541, "dur": 0.067,
    "args": {
      "External id": 2489988,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11547
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040681.104, "dur": 0.069,
    "args": {
      "External id": 2489989,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11548
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040681.657, "dur": 0.069,
    "args": {
      "External id": 2489990,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11549
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040682.101, "dur": 0.068,
    "args": {
      "External id": 2489991,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11550
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040682.470, "dur": 0.070,
    "args": {
      "External id": 2489992,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11551
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040682.881, "dur": 0.056,
    "args": {
      "External id": 2489993,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11552
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040683.330, "dur": 0.064,
    "args": {
      "External id": 2489994,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11553
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040683.697, "dur": 0.061,
    "args": {
      "External id": 2489995,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11554
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040684.058, "dur": 0.066,
    "args": {
      "External id": 2489996,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11555
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040684.474, "dur": 0.070,
    "args": {
      "External id": 2489997,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11556
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040684.794, "dur": 0.061,
    "args": {
      "External id": 2489998,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11557
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040685.595, "dur": 0.071,
    "args": {
      "External id": 2489999,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11558
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040686.043, "dur": 0.052,
    "args": {
      "External id": 2490000,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11559
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040686.588, "dur": 0.061,
    "args": {
      "External id": 2490001,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11560
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040686.895, "dur": 0.047,
    "args": {
      "External id": 2490002,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11561
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040687.386, "dur": 0.056,
    "args": {
      "External id": 2490003,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11562
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040687.693, "dur": 0.050,
    "args": {
      "External id": 2490004,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11563
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040688.321, "dur": 0.058,
    "args": {
      "External id": 2490005,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11564
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040688.626, "dur": 0.047,
    "args": {
      "External id": 2490006,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11565
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040689.142, "dur": 0.060,
    "args": {
      "External id": 2490007,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11566
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040689.446, "dur": 0.051,
    "args": {
      "External id": 2490008,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11567
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040690.439, "dur": 0.069,
    "args": {
      "External id": 2490009,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11568
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040690.808, "dur": 0.058,
    "args": {
      "External id": 2490010,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11569
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040691.450, "dur": 0.060,
    "args": {
      "External id": 2490011,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11570
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040691.770, "dur": 0.052,
    "args": {
      "External id": 2490012,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11571
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040692.312, "dur": 0.063,
    "args": {
      "External id": 2490013,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11572
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040692.641, "dur": 0.049,
    "args": {
      "External id": 2490014,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11573
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040693.352, "dur": 0.073,
    "args": {
      "External id": 2490015,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11574
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040693.674, "dur": 0.054,
    "args": {
      "External id": 2490016,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11575
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040694.298, "dur": 0.058,
    "args": {
      "External id": 2490017,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11576
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040694.602, "dur": 0.050,
    "args": {
      "External id": 2490018,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11577
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040695.226, "dur": 0.066,
    "args": {
      "External id": 2490019,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11578
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040695.532, "dur": 0.051,
    "args": {
      "External id": 2490020,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11579
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040696.041, "dur": 0.067,
    "args": {
      "External id": 2490021,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11580
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040696.358, "dur": 0.053,
    "args": {
      "External id": 2490022,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11581
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040697.026, "dur": 0.064,
    "args": {
      "External id": 2490023,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11582
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040697.335, "dur": 0.057,
    "args": {
      "External id": 2490024,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11583
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040697.829, "dur": 0.091,
    "args": {
      "External id": 2490025,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11584
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040698.166, "dur": 0.053,
    "args": {
      "External id": 2490026,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11585
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040698.706, "dur": 0.066,
    "args": {
      "External id": 2490027,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11586
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040699.023, "dur": 0.052,
    "args": {
      "External id": 2490028,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11587
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040699.545, "dur": 0.066,
    "args": {
      "External id": 2490029,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11588
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040699.872, "dur": 0.054,
    "args": {
      "External id": 2490030,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11589
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040700.514, "dur": 0.065,
    "args": {
      "External id": 2490031,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11590
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040700.829, "dur": 0.050,
    "args": {
      "External id": 2490032,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11591
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040701.508, "dur": 0.067,
    "args": {
      "External id": 2490033,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11592
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040701.824, "dur": 0.051,
    "args": {
      "External id": 2490034,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11593
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040702.318, "dur": 0.067,
    "args": {
      "External id": 2490035,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11594
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040702.629, "dur": 0.053,
    "args": {
      "External id": 2490036,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11595
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040703.397, "dur": 0.071,
    "args": {
      "External id": 2490037,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11596
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040703.714, "dur": 0.054,
    "args": {
      "External id": 2490038,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11597
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040704.342, "dur": 0.068,
    "args": {
      "External id": 2490039,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11598
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040704.651, "dur": 0.057,
    "args": {
      "External id": 2490040,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11599
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040705.264, "dur": 0.061,
    "args": {
      "External id": 2490041,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11600
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040705.569, "dur": 0.059,
    "args": {
      "External id": 2490042,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11601
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040706.192, "dur": 0.052,
    "args": {
      "External id": 2490043,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11602
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040706.492, "dur": 0.050,
    "args": {
      "External id": 2490044,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11603
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040707.044, "dur": 0.068,
    "args": {
      "External id": 2490045,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11604
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040707.358, "dur": 0.054,
    "args": {
      "External id": 2490046,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11605
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040707.975, "dur": 0.053,
    "args": {
      "External id": 2490047,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11606
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040708.275, "dur": 0.056,
    "args": {
      "External id": 2490048,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11607
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040708.949, "dur": 0.065,
    "args": {
      "External id": 2490049,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11608
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040709.255, "dur": 0.056,
    "args": {
      "External id": 2490050,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11609
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040709.712, "dur": 0.067,
    "args": {
      "External id": 2490051,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11610
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040710.028, "dur": 0.054,
    "args": {
      "External id": 2490052,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11611
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040710.653, "dur": 0.065,
    "args": {
      "External id": 2490053,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11612
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040710.962, "dur": 0.055,
    "args": {
      "External id": 2490054,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11613
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040711.564, "dur": 0.055,
    "args": {
      "External id": 2490055,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11614
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040711.861, "dur": 0.054,
    "args": {
      "External id": 2490056,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11615
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040712.478, "dur": 0.065,
    "args": {
      "External id": 2490057,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11616
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040712.785, "dur": 0.052,
    "args": {
      "External id": 2490058,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11617
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040713.491, "dur": 0.068,
    "args": {
      "External id": 2490059,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11618
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040713.797, "dur": 0.055,
    "args": {
      "External id": 2490060,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11619
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040714.292, "dur": 0.070,
    "args": {
      "External id": 2490061,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11620
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040714.608, "dur": 0.058,
    "args": {
      "External id": 2490062,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11621
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040715.315, "dur": 0.064,
    "args": {
      "External id": 2490063,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11622
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040715.626, "dur": 0.058,
    "args": {
      "External id": 2490064,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11623
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040716.256, "dur": 0.054,
    "args": {
      "External id": 2490065,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11624
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040716.558, "dur": 0.048,
    "args": {
      "External id": 2490066,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11625
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040717.065, "dur": 0.059,
    "args": {
      "External id": 2490067,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11626
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040717.365, "dur": 0.050,
    "args": {
      "External id": 2490068,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11627
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040717.811, "dur": 0.064,
    "args": {
      "External id": 2490069,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11628
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040718.121, "dur": 0.049,
    "args": {
      "External id": 2490070,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11629
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040718.835, "dur": 0.057,
    "args": {
      "External id": 2490071,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11630
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040719.164, "dur": 0.060,
    "args": {
      "External id": 2490072,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11631
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336754, "tid": 1336754,
    "ts": 1514658040719.693, "dur": 0.053,
    "args": {
      "External id": 2490073,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11632
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_fused_adamw_", "pid": 1336754, "tid": 1336754,
    "ts": 1514658041298.316, "dur": 3414.685,
    "args": {
      "External id": 2490074,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "0.0001072708166170532", "0.90000000000000002", "0.94999999999999996", "0.10000000000000001", "1.0000000000000001e-15", "False", "False", "", ""], "Input type": ["TensorList", "TensorList", "TensorList", "TensorList", "TensorList", "TensorList", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 11633
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_fused_adamw_", "pid": 1336754, "tid": 1336754,
    "ts": 1514658044210.221, "dur": 351.248,
    "args": {
      "External id": 2490075,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "0.0001072708166170532", "0.90000000000000002", "0.94999999999999996", "0.10000000000000001", "1.0000000000000001e-15", "False", "False", "", ""], "Input type": ["TensorList", "TensorList", "TensorList", "TensorList", "TensorList", "TensorList", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 11634
    }
  },
  {
    "name": "process_name", "ph": "M", "ts": 1514655936768.842, "pid": 1336754, "tid": 0,
    "args": {
      "name": "python3.12"
    }
  },
  {
    "name": "process_labels", "ph": "M", "ts": 1514655936768.842, "pid": 1336754, "tid": 0,
    "args": {
      "labels": "CPU"
    }
  },
  {
    "name": "process_sort_index", "ph": "M", "ts": 1514655936768.842, "pid": 1336754, "tid": 0,
    "args": {
      "sort_index": 1336754
    }
  },
  {
    "name": "thread_name", "ph": "M", "ts": 1514655936768.842, "pid": 1336754, "tid": 1381179,
    "args": {
      "name": "thread 1381179 (pt_autograd_1)"
    }
  },
  {
    "name": "thread_sort_index", "ph": "M", "ts": 1514655936768.842, "pid": 1336754, "tid": 1381179,
    "args": {
      "sort_index": 1381179
    }
  },
  {
    "name": "thread_name", "ph": "M", "ts": 1514655936768.842, "pid": 1336754, "tid": 1381179,
    "args": {
      "name": "thread 1381179 (python3.12)"
    }
  },
  {
    "name": "thread_sort_index", "ph": "M", "ts": 1514655936768.842, "pid": 1336754, "tid": 1381179,
    "args": {
      "sort_index": 1381179
    }
  },
  {
    "name": "thread_name", "ph": "M", "ts": 1514655936768.842, "pid": 1336754, "tid": 1336754,
    "args": {
      "name": "thread 1336754 (python3.12)"
    }
  },
  {
    "name": "thread_sort_index", "ph": "M", "ts": 1514655936768.842, "pid": 1336754, "tid": 1336754,
    "args": {
      "sort_index": 1336754
    }
  },
  {
    "ph": "X", "cat": "Trace", "ts": 1514655936693.357, "dur": 2109767.896,
    "pid": "Spans", "tid": "PyTorch Profiler",
    "name": "PyTorch Profiler (0)",
    "args": {
      "Op count": 0
    }
  },
  {
    "name": "process_sort_index", "ph": "M", "ts": 1514655936693.357,
    "pid": "Spans", "tid": 0,
    "args": {
      "sort_index": 536870912
    }
  },
  {
    "name": "Iteration Start: PyTorch Profiler", "ph": "i", "s": "g",
    "pid": "Traces", "tid": "Trace PyTorch Profiler", "ts": 1514655936693.357
  },
  {
    "name": "Record Window End", "ph": "i", "s": "g",
    "pid": "", "tid": "", "ts": 1514658108690.769
  }
  ],
  "traceName": "exp/mtp.1B.batch16.seqlen4096.context4096.warmup2000.update1.steps200000.lr2e-4.cosine/profile_trace/iteration_102912/rank1_trace.json",
  "displayTimeUnit": "ms",
  "baseTimeNanoseconds": 1751410836000000000
}